From 454f609aa6cb057772f56fdb7533ab9036ebd2ed Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Fri, 12 Nov 2021 14:18:32 +0800
Subject: [PATCH 01/16] build: emqx package name scheme include otp version

---
 Makefile                     | 1 +
 build                        | 5 ++++-
 deploy/packages/deb/Makefile | 2 +-
 deploy/packages/rpm/Makefile | 2 +-
 scripts/get-otp-vsn.sh       | 7 +++++++
 5 files changed, 14 insertions(+), 3 deletions(-)
 create mode 100755 scripts/get-otp-vsn.sh

diff --git a/Makefile b/Makefile
index 17ceb9481..11edd8929 100644
--- a/Makefile
+++ b/Makefile
@@ -3,6 +3,7 @@ REBAR_VERSION = 3.14.3-emqx-8
 REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
+export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
 export EMQX_DESC ?= EMQ X
 export EMQX_CE_DASHBOARD_VERSION ?= v4.3.3
diff --git a/build b/build
index be4f88672..dbbfcbdfb 100755
--- a/build
+++ b/build
@@ -12,6 +12,9 @@ ARTIFACT="$2"
 # ensure dir
 cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
 
+OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}"
+export OTP_VSN
+
 PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
 export PKG_VSN
 
@@ -116,7 +119,7 @@ make_zip() {
         log "ERROR: $tarball is not found"
     fi
     local zipball
-    zipball="${pkgpath}/${PROFILE}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip"
+    zipball="${pkgpath}/${PROFILE}-${OTP_VSN}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip"
     tar zxf "${tarball}" -C "${tard}/emqx"
     ## try to be portable for zip packages.
     ## for DEB and RPM packages the dependencies are resoved by yum and apt
diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile
index 48124d780..6f3956409 100644
--- a/deploy/packages/deb/Makefile
+++ b/deploy/packages/deb/Makefile
@@ -8,7 +8,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
 
 TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
 SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture)
-TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH)
+TARGET_PKG := $(EMQX_NAME)-$(OTP_VSN)-$(SYSTEM)-$(PKG_VSN)-$(ARCH)
 
 .PHONY: all
 all: | $(BUILT)
diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile
index 5a6e6bee4..e8c01e935 100644
--- a/deploy/packages/rpm/Makefile
+++ b/deploy/packages/rpm/Makefile
@@ -16,7 +16,7 @@ endif
 EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
 
 TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
-TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH)
+TARGET_PKG := $(EMQX_NAME)-$(OTP_VSN)-$(SYSTEM)-$(PKG_VSN)-$(ARCH)
 ifeq ($(RPM_REL),)
 	# no tail
 	RPM_REL := 1
diff --git a/scripts/get-otp-vsn.sh b/scripts/get-otp-vsn.sh
new file mode 100755
index 000000000..fd5b4fd0b
--- /dev/null
+++ b/scripts/get-otp-vsn.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -euo pipefail
+
+vsn=$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell)
+
+echo "otp${vsn}"

From 7e91a47be8b1ae68ded8d7ddb22350be8a8eebc9 Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Fri, 12 Nov 2021 14:50:56 +0800
Subject: [PATCH 02/16] ci: update workflows

---
 .github/workflows/build_packages.yaml | 53 +++++++++++----------------
 1 file changed, 22 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index befa37912..efaf14bfc 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -212,6 +212,8 @@ jobs:
     strategy:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+        otp:
+          - 23.3.4.8-1
         arch:
           - amd64
           - arm64
@@ -248,15 +250,11 @@ jobs:
         shell: bash
 
     steps:
-    - name: prepare docker
-      run: |
-        mkdir -p $HOME/.docker
-        echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
-        echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50}' | sudo tee /etc/docker/daemon.json
-        sudo systemctl restart docker
-        docker info
-        docker buildx create --use --name mybuild
-        docker run --rm --privileged tonistiigi/binfmt --install all
+    - uses: docker/setup-buildx-action@v1
+    - uses: docker/setup-qemu-action@v1
+      with:
+        image: tonistiigi/binfmt:latest
+        platforms: all
     - uses: actions/download-artifact@v2
       with:
         name: source
@@ -265,6 +263,7 @@ jobs:
       run: unzip -q source.zip
     - name: downloads old emqx zip packages
       env:
+        OTP_VSN: otp${{ matrix.otp }}
         PROFILE: ${{ matrix.profile }}
         ARCH: ${{ matrix.arch }}
         SYSTEM: ${{ matrix.os }}
@@ -283,36 +282,28 @@ jobs:
         cd source/_upgrade_base
         old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
         for tag in ${old_vsns[@]}; do
-          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256
-            echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1
+          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip) | grep -oE "^[23]+")" ];then
+            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip
+            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip.sha256
+            echo "$(cat ${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip.sha256) ${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip" | sha256sum -c || exit 1
           fi
         done
     - name: build emqx packages
       env:
-        ERL_OTP: erl23.2.7.2-emqx-2
+        OTP: ${{ matrix.otp }}
         PROFILE: ${{ matrix.profile }}
         ARCH: ${{ matrix.arch }}
         SYSTEM: ${{ matrix.os }}
+      working-directory: source
       run: |
-        set -e -u
-        cd source
-        docker buildx build --no-cache \
-          --platform=linux/$ARCH \
-          -t cross_build_emqx_for_$SYSTEM \
-          -f .ci/build_packages/Dockerfile \
-          --build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-$SYSTEM \
-          --build-arg EMQX_NAME=$PROFILE \
-          --output type=tar,dest=/tmp/cross-build-$PROFILE-for-$SYSTEM.tar .
-
-        mkdir -p /tmp/packages/$PROFILE
-        tar -xvf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar --wildcards emqx/_packages/$PROFILE/*
-        mv emqx/_packages/$PROFILE/* /tmp/packages/$PROFILE/
-        rm -rf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar
-
-        docker rm -f $(docker ps -a -q)
-        docker volume prune -f
+        docker run -i --rm \
+        -v $(pwd):/emqx \
+        --workdir /emqx \
+        --platform linux/$ARCH \
+        ghcr.io/emqx/emqx-builder/4.4:$OTP-$SYSTEM \
+        bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \
+                   make $PROFILE-pkg || cat rebar3.crashdump; \
+                   EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh"
     - name: create sha256
       env:
         PROFILE: ${{ matrix.profile}}

From cace53a02b8b9e535be158955029db6c8c19abac Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Fri, 12 Nov 2021 17:24:25 +0800
Subject: [PATCH 03/16] build: update docker build

---
 .github/workflows/build_packages.yaml     |  67 ++++----
 .github/workflows/run_automate_tests.yaml |  59 +++----
 .github/workflows/run_fvt_tests.yaml      |  15 +-
 Makefile                                  |  11 +-
 build                                     |  17 ++
 deploy/docker/Dockerfile                  |  21 +--
 docker.mk                                 | 188 ----------------------
 7 files changed, 104 insertions(+), 274 deletions(-)
 delete mode 100644 docker.mk

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index efaf14bfc..4e696dab2 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -308,8 +308,8 @@ jobs:
       env:
         PROFILE: ${{ matrix.profile}}
       run: |
-        if [ -d /tmp/packages/$PROFILE ]; then
-          cd /tmp/packages/$PROFILE
+        if [ -d _packages/$PROFILE ]; then
+          cd _packages/$PROFILE
             for var in $(ls emqx-* ); do
               bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"
             done
@@ -319,7 +319,7 @@ jobs:
       if: startsWith(github.ref, 'refs/tags/')
       with:
         name: ${{ matrix.profile }}
-        path: /tmp/packages/${{ matrix.profile }}/.
+        path: _packages/${{ matrix.profile }}/.
 
   docker:
     runs-on: ubuntu-20.04
@@ -329,6 +329,8 @@ jobs:
     strategy:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+        otp:
+          - 23.3.4.8-1
         arch:
           - [amd64, x86_64]
           - [arm64v8, aarch64]
@@ -348,22 +350,42 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - name: build emqx docker image
-      env:
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch[0] }}
-        QEMU_ARCH: ${{ matrix.arch[1] }}
-      run: |
-        sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-
-        cd source
-        sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH  make docker
-        cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
-    - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
+    - uses: docker/setup-buildx-action@v1
+    - uses: docker/setup-qemu-action@v1
       with:
-        name: ${{ matrix.profile }}
-        path: source/_packages/${{ matrix.profile }}/.
+        image: tonistiigi/binfmt:latest
+        platforms: all
+    - uses: docker/metadata-action@v3
+      id: meta
+      with:
+        images: ${{ github.repository_owner }}/${{ matrix.profile }}
+        flavor: |
+          latest=${{ !github.event.release.prerelease }}
+        tags: |
+          type=ref,event=branch
+          type=ref,event=pr
+          type=ref,event=tag
+          type=semver,pattern={{version}}-otp${{ matrix.otp }}
+          type=semver,pattern={{major}}.{{minor}}-otp${{ matrix.otp }}
+    - uses: docker/login-action@v1
+      if: github.event_name == 'release'
+      with:
+        username: ${{ secrets.DOCKER_HUB_USER }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
+    - uses: docker/build-push-action@v2
+      with:
+        push: ${{ github.event_name == 'release' }}
+        pull: true
+        no-cache: true
+        platforms: linux/amd64,linux/arm64
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+        build-args: |
+          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4:${{ matrix.otp }}-alpine3.14
+          RUN_FROM=alpine:3.14
+          EMQX_NAME=${{ matrix.profile }}
+        file: source/deploy/docker/Dockerfile
+        context: source
 
   delete-artifact:
     runs-on: ubuntu-20.04
@@ -443,15 +465,6 @@ jobs:
              -X POST \
              -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
              ${{ secrets.EMQX_IO_RELEASE_API }}
-    - name: push docker image to docker hub
-      if: github.event_name == 'release'
-      run: |
-        set -e -x -u
-        sudo make docker-prepare
-        cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd -
-        echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
-        sudo TARGET=emqx/${{ matrix.profile }} make docker-push
-        sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list
     - name: update repo.emqx.io
       if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
       run: |
diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml
index fb213c713..4bea3f128 100644
--- a/.github/workflows/run_automate_tests.yaml
+++ b/.github/workflows/run_automate_tests.yaml
@@ -10,14 +10,19 @@ on:
 
 jobs:
   build:
+    strategy:
+      matrix:
+        otp:
+          - 23.3.4.8-1
+
     runs-on: ubuntu-latest
     outputs:
-      imgname: ${{ steps.build_docker.outputs.imgname}}
-      version: ${{ steps.build_docker.outputs.version}}
+      imgname: ${{ steps.prepare.outputs.imgname}}
+      version: ${{ steps.prepare.outputs.version}}
     steps:
     - uses: actions/checkout@v2
-    - name: build docker
-      id: build_docker
+    - name: prepare
+      id: prepare
       run: |
         if [ -f EMQX_ENTERPRISE ]; then
           echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
@@ -25,18 +30,20 @@ jobs:
           echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
           make deps-emqx-ee
           make clean
-        fi
-        make docker
-        echo "::set-output name=version::$(./pkg-vsn.sh)"
-        if [ -f EMQX_ENTERPRISE ]; then
           echo "::set-output name=imgname::emqx-ee"
+          echo "::set-output name=version::$(./pkg-vsn.sh)"
         else
           echo "::set-output name=imgname::emqx"
+          echo "::set-output name=version::$(./pkg-vsn.sh)"
         fi
+    - name: build docker image
+      run: |
+        make ${{ steps.prepare.outputs.imgname }}-docker
+        docker save emqx/${{ steps.prepare.outputs.imgname }}:${{ steps.prepare.outputs.version }} -o image.tar.gz
     - uses: actions/upload-artifact@v2
       with:
-        name: emqx-docker-image-zip
-        path: _packages/${{ steps.build_docker.outputs.imgname }}/${{ steps.build_docker.outputs.imgname }}-docker-${{ steps.build_docker.outputs.version }}.zip
+        name: image
+        path: image.tar.gz
 
   webhook:
     runs-on: ubuntu-latest
@@ -52,15 +59,11 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/download-artifact@v2
       with:
-        name: emqx-docker-image-zip
+        name: image
         path: /tmp
     - name: load docker image
-      env:
-        imgname: ${{ needs.build.outputs.imgname}}
-        version: ${{ needs.build.outputs.version }}
       run: |
-        unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
-        docker load < /tmp/${imgname}-docker-${version}
+        docker load < /tmp/image.tar.gz
     - name: docker compose up
       timeout-minutes: 5
       env:
@@ -152,15 +155,11 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/download-artifact@v2
       with:
-        name: emqx-docker-image-zip
+        name: image
         path: /tmp
     - name: load docker image
-      env:
-        imgname: ${{ needs.build.outputs.imgname }}
-        version: ${{ needs.build.outputs.version }}
       run: |
-        unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
-        docker load < /tmp/${imgname}-docker-${version}
+        docker load < /tmp/image.tar.gz
     - name: docker compose up
       timeout-minutes: 5
       env:
@@ -259,15 +258,11 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/download-artifact@v2
       with:
-        name: emqx-docker-image-zip
+        name: image
         path: /tmp
     - name: load docker image
-      env:
-        imgname: ${{ needs.build.outputs.imgname }}
-        version: ${{ needs.build.outputs.version }}
       run: |
-        unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
-        docker load < /tmp/${imgname}-docker-${version}
+        docker load < /tmp/image.tar.gz
     - name: docker compose up
       timeout-minutes: 5
       env:
@@ -355,15 +350,11 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/download-artifact@v2
       with:
-        name: emqx-docker-image-zip
+        name: image
         path: /tmp
     - name: load docker image
-      env:
-        imgname: ${{ needs.build.outputs.imgname }}
-        version: ${{ needs.build.outputs.version }}
       run: |
-        unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
-        docker load < /tmp/${imgname}-docker-${version}
+        docker load < /tmp/image.tar.gz
     - name: docker compose up
       timeout-minutes: 5
       env:
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index 80fced0e4..8eecc9528 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -25,13 +25,15 @@ jobs:
               echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
+              echo "PROFILE=emqx-ee" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
             else
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
+              echo "PROFILE=emqx" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
             fi
         - name: make emqx image
-          run: make docker
+          run: make ${PROFILE}-docker
         - name: run emqx
           timeout-minutes: 5
           run: |
@@ -79,11 +81,15 @@ jobs:
               echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
+              echo "PROFILE=emqx-ee" >> $GITHUB_ENV
+              echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
             else
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
+              echo "PROFILE=emqx" >> $GITHUB_ENV
+              echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
             fi
         - name: make emqx image
-          run: make docker
+          run: make ${PROFILE}-docker
         - name: install k3s
           env:
             KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
@@ -105,11 +111,10 @@ jobs:
             KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
           timeout-minutes: 5
           run: |
-            version=$(./pkg-vsn.sh)
-            sudo docker save ${TARGET}:$version -o emqx.tar.gz
+            sudo docker save ${TARGET}:${EMQX_TAG} -o emqx.tar.gz
             sudo k3s ctr image import emqx.tar.gz
 
-            sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/charts/emqx/Chart.yaml
+            sed -i -r "s/^appVersion: .*$/appVersion: \"${EMQX_TAG}\"/g" deploy/charts/emqx/Chart.yaml
             sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml
 
             helm install emqx \
diff --git a/Makefile b/Makefile
index 11edd8929..e758892e9 100644
--- a/Makefile
+++ b/Makefile
@@ -145,11 +145,18 @@ $1: $1-rel
 endef
 $(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt))))
 
+## docker target is to create docker instructions
+.PHONY: $(REL_PROFILES:%=%-docker)
+define gen-docker-target
+$1-docker: $(COMMON_DEPS)
+	@$(BUILD) $1 docker
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
+
 .PHONY: run
 run: $(PROFILE) quickrun
 
 .PHONY: quickrun
 quickrun:
 	./_build/$(PROFILE)/rel/emqx/bin/emqx console
-
-include docker.mk
diff --git a/build b/build
index dbbfcbdfb..7caf2579d 100755
--- a/build
+++ b/build
@@ -127,6 +127,20 @@ make_zip() {
     (cd "${tard}" && zip -qr - emqx) > "${zipball}"
 }
 
+make_docker() {
+    ## Build Docker image
+    echo "DOCKER BUILD: Build Docker image."
+    echo "DOCKER BUILD: build version -> $PKG_VSN."
+    echo "DOCKER BUILD: docker repo -> emqx/$PROFILE "
+
+    docker build --no-cache \
+       --build-arg BUILD_FROM="ghcr.io/emqx/emqx-builder/4.4:${OTP:-23.3.4.8-1}-alpine3.14" \
+       --build-arg RUN_FROM="alpine:3.14" \
+       --build-arg EMQX_NAME="$PROFILE" \
+       --tag "emqx/$PROFILE:$PKG_VSN" \
+       -f deploy/docker/Dockerfile .
+}
+
 log "building artifact=$ARTIFACT for profile=$PROFILE"
 
 case "$ARTIFACT" in
@@ -147,6 +161,9 @@ case "$ARTIFACT" in
         make -C "deploy/packages/${PKGERDIR}" clean
         EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}"
         ;;
+    docker)
+        make_docker
+        ;;
     *)
         log "Unknown artifact $ARTIFACT"
         exit 1
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index e362c6b73..ae11ad424 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -1,10 +1,7 @@
-ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine-amd64
-ARG RUN_FROM=alpine:3.12
+ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4:23.3.4.8-1-alpine3.14
+ARG RUN_FROM=alpine:3.14
 FROM ${BUILD_FROM} AS builder
 
-ARG QEMU_ARCH=x86_64
-COPY tmp/qemu-$QEMU_ARCH-stati* /usr/bin/
-
 RUN apk add --no-cache \
     git \
     curl \
@@ -32,21 +29,9 @@ RUN cd /emqx \
 
 FROM $RUN_FROM
 
-# Basic build-time metadata as defined at http://label-schema.org
-LABEL org.label-schema.docker.dockerfile="Dockerfile" \
-    org.label-schema.license="GNU" \
-    org.label-schema.name="emqx" \
-    org.label-schema.version=${PKG_VSN} \
-    org.label-schema.description="EMQ (Erlang MQTT Broker) is a distributed, massively scalable, highly extensible MQTT messaging broker written in Erlang/OTP." \
-    org.label-schema.url="https://emqx.io" \
-    org.label-schema.vcs-type="Git" \
-    org.label-schema.vcs-url="https://github.com/emqx/emqx" \
-    maintainer="EMQ X Team <support@emqx.io>"
-
-ARG QEMU_ARCH=x86_64
 ARG EMQX_NAME=emqx
 
-COPY deploy/docker/docker-entrypoint.sh tmp/qemu-$QEMU_ARCH-stati* /usr/bin/
+COPY deploy/docker/docker-entrypoint.sh /usr/bin/
 COPY --from=builder /emqx/_build/$EMQX_NAME/rel/emqx /opt/emqx
 
 RUN ln -s /opt/emqx/bin/* /usr/local/bin/
diff --git a/docker.mk b/docker.mk
deleted file mode 100644
index 22f2b6e4f..000000000
--- a/docker.mk
+++ /dev/null
@@ -1,188 +0,0 @@
-#!/usr/bin/make -f
-# -*- makefile -*-
-
-## default globals.
-## when built with `make docker` command the default profile is either emqx or emqx-ee (for enterprise)
-## or the TARGET varialbe can be set beforehand to force a different name
-TARGET ?= emqx/$(PROFILE)
-QEMU_ARCH ?= x86_64
-ARCH ?= amd64
-QEMU_VERSION ?= v5.0.0-2
-OS ?= alpine
-export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
-
-ifeq ($(findstring emqx-ee, $(TARGET)), emqx-ee)
-	ARCH_LIST := amd64 arm64v8 arm32v7
-	EMQX_NAME := emqx-ee
-else ifeq ($(findstring emqx-edge, $(TARGET)), emqx-edge)
-	ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x
-	EMQX_NAME := emqx-edge
-else
-	ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x
-	EMQX_NAME := emqx
-endif
-
-.PHONY: docker
-docker: docker-build docker-tag docker-save
-
-.PHONY: docker-prepare
-docker-prepare:
-	## Prepare the machine before any code installation scripts
-	# @echo "PREPARE: Setting up dependencies."
-	# @apt update -y
-	# @apt install --only-upgrade docker-ce -y
-
-	## Update docker configuration to enable docker manifest command
-	@echo "PREPARE: Updating docker configuration"
-	@mkdir -p $$HOME/.docker
-
-	# enable experimental to use docker manifest command
-	@echo '{ "experimental": "enabled" }' | tee $$HOME/.docker/config.json
-	# enable experimental
-	@echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50 }' | tee /etc/docker/daemon.json
-	@service docker restart
-
-.PHONY: docker-build
-docker-build:
-	## Build Docker image
-	@echo "DOCKER BUILD: Build Docker image."
-	@echo "DOCKER BUILD: build version -> $(PKG_VSN)."
-	@echo "DOCKER BUILD: arch - $(ARCH)."
-	@echo "DOCKER BUILD: qemu arch - $(QEMU_ARCH)."
-	@echo "DOCKER BUILD: docker repo - $(TARGET) "
-	@echo "DOCKER BUILD: emqx name - $(EMQX_NAME)."
-
-	## Prepare qemu to build images other then x86_64 on travis
-	@echo "PREPARE: Qemu" \
-	&& docker run --rm --privileged multiarch/qemu-user-static:register --reset
-
-	@mkdir -p tmp \
-	&& cd tmp \
-	&& curl -L -o qemu-$(QEMU_ARCH)-static.tar.gz https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-$(QEMU_ARCH)-static.tar.gz \
-	&& tar xzf qemu-$(QEMU_ARCH)-static.tar.gz \
-	&& cd -
-
-	@docker build --no-cache \
-		--build-arg PKG_VSN=$(PKG_VSN)  \
-		--build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine-$(ARCH)  \
-		--build-arg RUN_FROM=$(ARCH)/alpine:3.12 \
-		--build-arg EMQX_NAME=$(EMQX_NAME) \
-		--build-arg QEMU_ARCH=$(QEMU_ARCH) \
-		--tag $(TARGET):build-$(OS)-$(ARCH) \
-		-f deploy/docker/Dockerfile .
-
-.PHONY: docker-tag
-docker-tag:
-	@echo "DOCKER TAG: Tag Docker image."
-	@for arch in $(ARCH_LIST); do \
-		if [ -n  "$$(docker images -q $(TARGET):build-$(OS)-$${arch})" ]; then \
-			docker tag $(TARGET):build-$(OS)-$${arch}  $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \
-			echo "DOCKER TAG: $(TARGET):$(PKG_VSN)-$(OS)-$${arch}"; \
-			if [ $${arch} = amd64 ]; then \
-				docker tag $(TARGET):$(PKG_VSN)-$(OS)-amd64 $(TARGET):$(PKG_VSN); \
-				echo "DOCKER TAG: $(TARGET):$(PKG_VSN)"; \
-			fi; \
-		fi; \
-	done
-
-.PHONY: docker-save
-docker-save:
-	@echo "DOCKER SAVE: Save Docker image."
-
-	@mkdir -p _packages/$(EMQX_NAME)
-
-	@if [ -n  "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \
-		docker save $(TARGET):$(PKG_VSN) > $(EMQX_NAME)-docker-$(PKG_VSN); \
-		zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN).zip $(EMQX_NAME)-docker-$(PKG_VSN); \
-		mv ./$(EMQX_NAME)-docker-$(PKG_VSN).zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN).zip; \
-	fi
-
-	@for arch in $(ARCH_LIST); do \
-		if [ -n  "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-			docker save  $(TARGET):$(PKG_VSN)-$(OS)-$${arch} > $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \
-			zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \
-			mv ./$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip; \
-		fi; \
-	done
-
-.PHONY: docker-push
-docker-push:
-	@echo "DOCKER PUSH: Push Docker image.";
-	@echo "DOCKER PUSH: pushing - $(TARGET):$(PKG_VSN).";
-
-	@if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \
-		docker push $(TARGET):$(PKG_VSN); \
-		docker tag $(TARGET):$(PKG_VSN) $(TARGET):latest; \
-		docker push $(TARGET):latest; \
-	fi;
-
-	@for arch in $(ARCH_LIST); do \
-		if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-			docker push  $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \
-		fi; \
-	done
-
-.PHONY: docker-manifest-list
-docker-manifest-list:
-	version="docker manifest create --amend $(TARGET):$(PKG_VSN)"; \
-	latest="docker manifest create --amend $(TARGET):latest"; \
-	for arch in $(ARCH_LIST); do \
-		if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ];then \
-			version="$${version} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \
-			latest="$${latest} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \
-		fi; \
-	done; \
-	eval $$version; \
-	eval $$latest;
-
-	for arch in $(ARCH_LIST); do \
-		case $${arch} in \
-			"amd64") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \
-				fi; \
-				;; \
-			"arm64v8") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \
-				fi; \
-				;; \
-			"arm32v7") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \
-				fi; \
-				;; \
-			"i386") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \
-				fi; \
-				;; \
-			"s390x") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \
-				fi; \
-				;; \
-		esac; \
-	done;
-
-	docker manifest inspect $(TARGET):$(PKG_VSN)
-	docker manifest push $(TARGET):$(PKG_VSN);
-	docker manifest inspect $(TARGET):latest
-	docker manifest push $(TARGET):latest;
-
-.PHONY: docker-clean
-docker-clean:
-	@echo "DOCKER CLEAN: Clean Docker image."
-
-	@if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN))" ]; then docker rmi -f $$(docker images -q  $(TARGET):$(PKG_VSN)); fi
-
-	@for arch in $(ARCH_LIST); do \
-		if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-			docker rmi -f $$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch}); \
-		fi \
-	done

From a6032d54352d975d1e169bd55c9384faf98c2d35 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Mon, 15 Nov 2021 23:02:28 +0100
Subject: [PATCH 04/16] build: update default otp versions

---
 .tool-versions           | 2 +-
 build                    | 6 ++++--
 deploy/docker/Dockerfile | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.tool-versions b/.tool-versions
index 200caa93b..a42915120 100644
--- a/.tool-versions
+++ b/.tool-versions
@@ -1 +1 @@
-erlang 23.2.7.2-emqx-2
+erlang 23.3.4.9-2
diff --git a/build b/build
index 7caf2579d..5cd022dcc 100755
--- a/build
+++ b/build
@@ -14,6 +14,8 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
 
 OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}"
 export OTP_VSN
+EMQX_RUNNER_IMAGE='alpine:3.14'
+EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE}}"
 
 PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
 export PKG_VSN
@@ -134,8 +136,8 @@ make_docker() {
     echo "DOCKER BUILD: docker repo -> emqx/$PROFILE "
 
     docker build --no-cache \
-       --build-arg BUILD_FROM="ghcr.io/emqx/emqx-builder/4.4:${OTP:-23.3.4.8-1}-alpine3.14" \
-       --build-arg RUN_FROM="alpine:3.14" \
+       --build-arg BUILD_FROM="${EMQX_BUILDER}" \
+       --build-arg RUN_FROM="{EMQX_RUNNER_IMAGE}" \
        --build-arg EMQX_NAME="$PROFILE" \
        --tag "emqx/$PROFILE:$PKG_VSN" \
        -f deploy/docker/Dockerfile .
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index ae11ad424..b48011bfd 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -1,4 +1,4 @@
-ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4:23.3.4.8-1-alpine3.14
+ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-alpine3.14
 ARG RUN_FROM=alpine:3.14
 FROM ${BUILD_FROM} AS builder
 

From f711f78c5c591dec92adee63a8cdd672b5c52903 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Mon, 15 Nov 2021 23:12:31 +0100
Subject: [PATCH 05/16] build: pin otp version 23.3.4.9-2

---
 .github/workflows/build_packages.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 4e696dab2..85d14bbfb 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -11,7 +11,7 @@ on:
 jobs:
   prepare:
     runs-on: ubuntu-20.04
-    container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
 
     outputs:
       profiles: ${{ steps.set_profile.outputs.profiles}}
@@ -134,7 +134,7 @@ jobs:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         erl_otp:
-          - 23.2.7.2-emqx-2
+          - 23.3.4.9-2
         exclude:
           - profile: emqx-edge
 
@@ -213,7 +213,7 @@ jobs:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         otp:
-          - 23.3.4.8-1
+          - 23.3.4.9-2
         arch:
           - amd64
           - arm64
@@ -300,7 +300,7 @@ jobs:
         -v $(pwd):/emqx \
         --workdir /emqx \
         --platform linux/$ARCH \
-        ghcr.io/emqx/emqx-builder/4.4:$OTP-$SYSTEM \
+        ghcr.io/emqx/emqx-builder/4.4-1:$OTP-$SYSTEM \
         bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \
                    make $PROFILE-pkg || cat rebar3.crashdump; \
                    EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh"
@@ -330,7 +330,7 @@ jobs:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         otp:
-          - 23.3.4.8-1
+          - 23.3.4.9-2
         arch:
           - [amd64, x86_64]
           - [arm64v8, aarch64]
@@ -381,7 +381,7 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4:${{ matrix.otp }}-alpine3.14
+          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:${{ matrix.otp }}-alpine3.14
           RUN_FROM=alpine:3.14
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile

From d350281270cae5fe9423edeaa9c3f7a6b2fbdd42 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Mon, 15 Nov 2021 23:33:23 +0100
Subject: [PATCH 06/16] build: pin
 ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04

---
 .ci/build_packages/Dockerfile                  | 2 +-
 .ci/docker-compose-file/docker-compose.yaml    | 2 +-
 .github/workflows/build_slim_packages.yaml     | 6 +++---
 .github/workflows/check_deps_integrity.yaml    | 2 +-
 .github/workflows/run_acl_migration_tests.yaml | 2 +-
 .github/workflows/run_fvt_tests.yaml           | 4 ++--
 .github/workflows/run_test_cases.yaml          | 4 ++--
 7 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile
index 6f9a12159..0a7c7496d 100644
--- a/.ci/build_packages/Dockerfile
+++ b/.ci/build_packages/Dockerfile
@@ -1,4 +1,4 @@
-ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
 FROM ${BUILD_FROM}
 
 ARG EMQX_NAME=emqx
diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml
index a502aeb6e..0415ac142 100644
--- a/.ci/docker-compose-file/docker-compose.yaml
+++ b/.ci/docker-compose-file/docker-compose.yaml
@@ -3,7 +3,7 @@ version: '3.9'
 services:
   erlang:
     container_name: erlang
-    image: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    image: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
     env_file:
       - conf.env
     environment:
diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index bf85578c5..5ee2a63dd 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -15,12 +15,12 @@ jobs:
     strategy:
       matrix:
         erl_otp:
-        - erl23.2.7.2-emqx-2
+        - 23.3.4.9-2
         os:
         - ubuntu20.04
         - centos7
 
-    container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }}
+    container: ghcr.io/emqx/emqx-builder/4.4-1:${{ matrix.erl_otp }}-${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v1
@@ -58,7 +58,7 @@ jobs:
     strategy:
       matrix:
         erl_otp:
-        - 23.2.7.2-emqx-2
+        - 23.3.4.9-2
 
     steps:
     - uses: actions/checkout@v1
diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml
index 0aa8f7903..5ebd2f69d 100644
--- a/.github/workflows/check_deps_integrity.yaml
+++ b/.github/workflows/check_deps_integrity.yaml
@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-20.04
-    container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
 
     steps:
       - uses: actions/checkout@v2
diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml
index 855d9463c..4cd2a914b 100644
--- a/.github/workflows/run_acl_migration_tests.yaml
+++ b/.github/workflows/run_acl_migration_tests.yaml
@@ -5,7 +5,7 @@ on: workflow_dispatch
 jobs:
     test:
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
         strategy:
             fail-fast: true
         env:
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index 8eecc9528..4bc0d13b2 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -185,7 +185,7 @@ jobs:
 
     relup_test_plan:
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
         outputs:
           profile: ${{ steps.profile-and-versions.outputs.profile }}
           vsn: ${{ steps.profile-and-versions.outputs.vsn }}
@@ -234,7 +234,7 @@ jobs:
     relup_test_build:
         needs: relup_test_plan
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
         defaults:
           run:
             shell: bash
diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 3fbe88c40..9d002138e 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -10,7 +10,7 @@ on:
 jobs:
     run_static_analysis:
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
 
         steps:
         - uses: actions/checkout@v2
@@ -27,7 +27,7 @@ jobs:
 
     run_proper_test:
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
 
         steps:
         - uses: actions/checkout@v2

From b68f01e7e718b4f4db66f6cad2d795f75a9b90d7 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Mon, 15 Nov 2021 23:42:19 +0100
Subject: [PATCH 07/16] build: do not print 'otp' prefix for otp version

---
 scripts/get-otp-vsn.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scripts/get-otp-vsn.sh b/scripts/get-otp-vsn.sh
index fd5b4fd0b..a791318dc 100755
--- a/scripts/get-otp-vsn.sh
+++ b/scripts/get-otp-vsn.sh
@@ -2,6 +2,4 @@
 
 set -euo pipefail
 
-vsn=$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell)
-
-echo "otp${vsn}"
+erl  -noshell -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().'

From c97c6aefc9dfb718a707a2322c82ac579dcd3e86 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Tue, 16 Nov 2021 00:46:58 +0100
Subject: [PATCH 08/16] build: rename zip package and rpm deb packages

---
 .ci/fvt_tests/local_relup_test_run.sh |  4 ++++
 .ci/fvt_tests/relup.lux               |  8 +++++---
 .github/workflows/build_packages.yaml | 11 ++++++-----
 .github/workflows/run_fvt_tests.yaml  |  2 ++
 build                                 |  9 ++++-----
 deploy/packages/deb/Makefile          |  2 +-
 deploy/packages/deb/debian/control    |  2 +-
 deploy/packages/rpm/Makefile          |  9 +++++----
 deploy/packages/rpm/emqx.spec         |  2 +-
 9 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/.ci/fvt_tests/local_relup_test_run.sh b/.ci/fvt_tests/local_relup_test_run.sh
index 7ef40b0f2..576ed6d02 100755
--- a/.ci/fvt_tests/local_relup_test_run.sh
+++ b/.ci/fvt_tests/local_relup_test_run.sh
@@ -15,6 +15,8 @@ PROFILE="$1"
 VSN="$2"
 OLD_VSN="$3"
 PACKAGE_PATH="$4"
+FROM_OTP_VSN="${5:-23.3.4.9-2}"
+TO_OTP_VSN="${6:-23.3.4.9-2}"
 
 TEMPDIR=$(mktemp -d)
 trap '{ rm -rf -- "$TEMPDIR"; }' EXIT
@@ -37,4 +39,6 @@ exec docker run \
         --var ONE_MORE_EMQX_PATH="/relup_test/one_more_emqx" \
         --var VSN="$VSN" \
         --var OLD_VSN="$OLD_VSN" \
+        --var FROM_OTP_VSN="$FROM_OTP_VSN" \
+        --var TO_OTP_VSN="$TO_OTP_VSN" \
         relup.lux
diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux
index 2940f5ce0..6da46a706 100644
--- a/.ci/fvt_tests/relup.lux
+++ b/.ci/fvt_tests/relup.lux
@@ -3,6 +3,8 @@
 [config var=ONE_MORE_EMQX_PATH]
 [config var=VSN]
 [config var=OLD_VSN]
+[config var=FROM_OTP_VSN]
+[config var=TO_OTP_VSN]
 
 [config shell_cmd=/bin/bash]
 [config timeout=600000]
@@ -19,7 +21,7 @@
 
 [shell emqx]
     !cd $PACKAGE_PATH
-    !unzip -q -o $PROFILE-ubuntu20.04-$(echo $OLD_VSN | sed  -r 's/[v|e]//g')-amd64.zip
+    !unzip -q -o $PROFILE-$(echo $OLD_VSN | sed  -r 's/[v|e]//g')-otp${FROM_OTP_VSN}-ubuntu20.04-amd64.zip
     ?SH-PROMPT
 
     !cd emqx
@@ -80,7 +82,7 @@
     !echo "" > log/emqx.log.1
     ?SH-PROMPT
 
-    !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
+    !cp -f ../$PROFILE-$VSN-otp${TO_OTP_VSN}-ubuntu20.04-amd64.zip releases/
 
     !./bin/emqx install $VSN
     ?Made release permanent: "$VSN"
@@ -105,7 +107,7 @@
     !echo "" > log/emqx.log.1
     ?SH-PROMPT
 
-    !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
+    !cp -f ../$PROFILE-$VSN-otp${TO_OTP_VSN}-ubuntu20.04-amd64.zip releases/
 
     !./bin/emqx install $VSN
     ?Made release permanent: "$VSN"
diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 85d14bbfb..f468c8c04 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -263,7 +263,7 @@ jobs:
       run: unzip -q source.zip
     - name: downloads old emqx zip packages
       env:
-        OTP_VSN: otp${{ matrix.otp }}
+        OTP_VSN: ${{ matrix.otp }}
         PROFILE: ${{ matrix.profile }}
         ARCH: ${{ matrix.arch }}
         SYSTEM: ${{ matrix.os }}
@@ -282,10 +282,11 @@ jobs:
         cd source/_upgrade_base
         old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
         for tag in ${old_vsns[@]}; do
-          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip) | grep -oE "^[23]+")" ];then
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip.sha256
-            echo "$(cat ${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip.sha256) ${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip" | sha256sum -c || exit 1
+          package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
+          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then
+            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip
+            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256
+            echo "$(cat $package_name.zip.sha256) $package_name.zip" | sha256sum -c || exit 1
           fi
         done
     - name: build emqx packages
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index 4bc0d13b2..82fc2b64c 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -321,6 +321,8 @@ jobs:
             --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
             --var VSN="$VSN" \
             --var OLD_VSN="$OLD_VSN" \
+            --var FROM_OTP_VSN="23.3.4.9-2" \
+            --var TO_OTP_VSN="23.3.4.9-2" \
             emqx_built/.ci/fvt_tests/relup.lux
         - uses: actions/upload-artifact@v2
           name: Save debug data
diff --git a/build b/build
index 5cd022dcc..50a2df1de 100755
--- a/build
+++ b/build
@@ -12,10 +12,9 @@ ARTIFACT="$2"
 # ensure dir
 cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
 
-OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}"
-export OTP_VSN
 EMQX_RUNNER_IMAGE='alpine:3.14'
-EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE}}"
+EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')"
+EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}"
 
 PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
 export PKG_VSN
@@ -81,7 +80,7 @@ make_relup() {
                 rm -rf "$tmp_dir"
             fi
             releases+=( "$base_vsn" )
-        done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-$SYSTEM*-$ARCH.zip" -type f)
+        done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f)
     fi
     if [ ${#releases[@]} -eq 0 ]; then
         log "No upgrade base found, relup ignored"
@@ -121,7 +120,7 @@ make_zip() {
         log "ERROR: $tarball is not found"
     fi
     local zipball
-    zipball="${pkgpath}/${PROFILE}-${OTP_VSN}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip"
+    zipball="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip"
     tar zxf "${tarball}" -C "${tard}/emqx"
     ## try to be portable for zip packages.
     ## for DEB and RPM packages the dependencies are resoved by yum and apt
diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile
index 6f3956409..1f709d860 100644
--- a/deploy/packages/deb/Makefile
+++ b/deploy/packages/deb/Makefile
@@ -8,7 +8,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
 
 TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
 SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture)
-TARGET_PKG := $(EMQX_NAME)-$(OTP_VSN)-$(SYSTEM)-$(PKG_VSN)-$(ARCH)
+TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
 
 .PHONY: all
 all: | $(BUILT)
diff --git a/deploy/packages/deb/debian/control b/deploy/packages/deb/debian/control
index e35535c12..b3794036c 100644
--- a/deploy/packages/deb/debian/control
+++ b/deploy/packages/deb/debian/control
@@ -4,7 +4,7 @@ Priority: optional
 Maintainer: emqx <contact@emqx.io>
 Build-Depends: debhelper (>=9)
 Standards-Version: 3.9.6
-Homepage: https://www.emqx.io
+Homepage: https://www.emqx.com
 
 Package: emqx
 Architecture: any
diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile
index e8c01e935..0e6cf9c5d 100644
--- a/deploy/packages/rpm/Makefile
+++ b/deploy/packages/rpm/Makefile
@@ -6,7 +6,7 @@ dash := -
 none :=
 space := $(none) $(none)
 RPM_VSN ?= $(shell echo $(PKG_VSN) | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?")
-RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]")
+RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]+")
 
 ARCH ?= amd64
 ifeq ($(ARCH),mips64)
@@ -16,12 +16,12 @@ endif
 EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
 
 TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
-TARGET_PKG := $(EMQX_NAME)-$(OTP_VSN)-$(SYSTEM)-$(PKG_VSN)-$(ARCH)
+TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
 ifeq ($(RPM_REL),)
 	# no tail
 	RPM_REL := 1
 endif
-SOURCE_PKG := emqx-$(SYSTEM)-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
+SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL)-otp$(OTP_VSN)-$(SYSTEM)-$(shell uname -m)
 
 SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi)
 # Not $(PWD) as it does not work for make -C
@@ -47,7 +47,8 @@ all: | $(BUILT)
 		--define "_service_dst $(SERVICE_DST)" \
 		--define "_post_addition $(POST_ADDITION)" \
 		--define "_preun_addition $(PREUN_ADDITION)" \
-		--define "_ostype -$(SYSTEM)" \
+		--define "_ostype $(SYSTEM)" \
+		--define "_otp_vsn $(OTP_VSN)" \
 		--define "_sharedstatedir /var/lib" \
 		emqx.spec
 	mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME)
diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec
index 882b7753e..0c309a8d0 100644
--- a/deploy/packages/rpm/emqx.spec
+++ b/deploy/packages/rpm/emqx.spec
@@ -5,7 +5,7 @@
 %define _log_dir %{_var}/log/%{_name}
 %define _lib_home /usr/lib/%{_name}
 %define _var_home %{_sharedstatedir}/%{_name}
-%define _build_name_fmt %{_arch}/%{_name}%{?_ostype}-%{_version}-%{_release}.%{_arch}.rpm
+%define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}-otp%{_otp_vsn}-%{?_ostype}.%{_arch}.rpm
 %define _build_id_links none
 
 Name: %{_package_name}

From 3f1fc64a98acfdccf05c17c1f16b67ff40480c22 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Tue, 16 Nov 2021 01:10:52 +0100
Subject: [PATCH 09/16] build: fix source rpm package name

---
 deploy/packages/rpm/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile
index 0e6cf9c5d..5ad80cd37 100644
--- a/deploy/packages/rpm/Makefile
+++ b/deploy/packages/rpm/Makefile
@@ -21,7 +21,7 @@ ifeq ($(RPM_REL),)
 	# no tail
 	RPM_REL := 1
 endif
-SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL)-otp$(OTP_VSN)-$(SYSTEM)-$(shell uname -m)
+SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL)-otp$(OTP_VSN)-$(SYSTEM).$(shell uname -m)
 
 SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi)
 # Not $(PWD) as it does not work for make -C

From 755dd11b070cf8f0d3909cc3d0cafc942cdb1df5 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Tue, 16 Nov 2021 01:28:42 +0100
Subject: [PATCH 10/16] build: always pull image before build

---
 build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build b/build
index 50a2df1de..08580de70 100755
--- a/build
+++ b/build
@@ -134,7 +134,7 @@ make_docker() {
     echo "DOCKER BUILD: build version -> $PKG_VSN."
     echo "DOCKER BUILD: docker repo -> emqx/$PROFILE "
 
-    docker build --no-cache \
+    docker build --no-cache --pull \
        --build-arg BUILD_FROM="${EMQX_BUILDER}" \
        --build-arg RUN_FROM="{EMQX_RUNNER_IMAGE}" \
        --build-arg EMQX_NAME="$PROFILE" \

From a178b6cc083617d24cdf5f50836a1af90b0a6e89 Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Tue, 16 Nov 2021 11:27:05 +0800
Subject: [PATCH 11/16] ci: fix make docker error for github action

---
 .github/workflows/run_automate_tests.yaml |  7 ++-----
 .github/workflows/run_fvt_tests.yaml      | 14 ++++++--------
 build                                     | 14 ++++++++------
 3 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml
index 4bea3f128..e4680be78 100644
--- a/.github/workflows/run_automate_tests.yaml
+++ b/.github/workflows/run_automate_tests.yaml
@@ -10,11 +10,6 @@ on:
 
 jobs:
   build:
-    strategy:
-      matrix:
-        otp:
-          - 23.3.4.8-1
-
     runs-on: ubuntu-latest
     outputs:
       imgname: ${{ steps.prepare.outputs.imgname}}
@@ -37,6 +32,8 @@ jobs:
           echo "::set-output name=version::$(./pkg-vsn.sh)"
         fi
     - name: build docker image
+      env:
+        OTP_VSN: 23.3.4.9-2
       run: |
         make ${{ steps.prepare.outputs.imgname }}-docker
         docker save emqx/${{ steps.prepare.outputs.imgname }}:${{ steps.prepare.outputs.version }} -o image.tar.gz
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index 82fc2b64c..2aece7312 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -13,10 +13,6 @@ jobs:
 
         steps:
         - uses: actions/checkout@v1
-        - uses: gleam-lang/setup-erlang@v1.1.2
-          id: install_erlang
-          with:
-            otp-version: 23.2
         - name: prepare
           run: |
             if make emqx-ee --dry-run > /dev/null 2>&1; then
@@ -24,6 +20,7 @@ jobs:
               git config --global credential.helper store
               echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
+              make clean
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
               echo "PROFILE=emqx-ee" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
@@ -33,6 +30,8 @@ jobs:
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
             fi
         - name: make emqx image
+          env:
+            OTP_VSN: 23.3.4.9-2
           run: make ${PROFILE}-docker
         - name: run emqx
           timeout-minutes: 5
@@ -69,10 +68,6 @@ jobs:
 
         steps:
         - uses: actions/checkout@v1
-        - uses: gleam-lang/setup-erlang@v1.1.2
-          id: install_erlang
-          with:
-            otp-version: 23.2
         - name: prepare
           run: |
             if make emqx-ee --dry-run > /dev/null 2>&1; then
@@ -80,6 +75,7 @@ jobs:
               git config --global credential.helper store
               echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
+              make clean
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
               echo "PROFILE=emqx-ee" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
@@ -89,6 +85,8 @@ jobs:
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
             fi
         - name: make emqx image
+          env:
+            OTP_VSN: 23.3.4.9-2
           run: make ${PROFILE}-docker
         - name: install k3s
           env:
diff --git a/build b/build
index 08580de70..f29dbc985 100755
--- a/build
+++ b/build
@@ -12,10 +12,6 @@ ARTIFACT="$2"
 # ensure dir
 cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
 
-EMQX_RUNNER_IMAGE='alpine:3.14'
-EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')"
-EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}"
-
 PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
 export PKG_VSN
 
@@ -129,14 +125,20 @@ make_zip() {
 }
 
 make_docker() {
+    EMQX_RUNNER_IMAGE='alpine:3.14'
+    EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')"
+    EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}"
+
     ## Build Docker image
     echo "DOCKER BUILD: Build Docker image."
-    echo "DOCKER BUILD: build version -> $PKG_VSN."
     echo "DOCKER BUILD: docker repo -> emqx/$PROFILE "
+    echo "DOCKER BUILD: build version -> $PKG_VSN."
+    echo "DOCKER BUILD: build from -> $EMQX_BUILDER."
+    echo "DOCKER BUILD: runner from -> $EMQX_RUNNER_IMAGE."
 
     docker build --no-cache --pull \
        --build-arg BUILD_FROM="${EMQX_BUILDER}" \
-       --build-arg RUN_FROM="{EMQX_RUNNER_IMAGE}" \
+       --build-arg RUN_FROM="${EMQX_RUNNER_IMAGE}" \
        --build-arg EMQX_NAME="$PROFILE" \
        --tag "emqx/$PROFILE:$PKG_VSN" \
        -f deploy/docker/Dockerfile .

From 2dc63cffea4f039c7b1132a437777efdd80ff168 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Tue, 16 Nov 2021 08:37:53 +0100
Subject: [PATCH 12/16] fix(emqx_misc): call gen_tcp to decide if ipv6_probe is
 supported

---
 src/emqx_misc.erl | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl
index 2b77d0455..01495460f 100644
--- a/src/emqx_misc.erl
+++ b/src/emqx_misc.erl
@@ -66,16 +66,9 @@ maybe_parse_ip(Host) ->
 
 %% @doc Add `ipv6_probe' socket option if it's supported.
 ipv6_probe(Opts) ->
-    case persistent_term:get({?MODULE, ipv6_probe_supported}, unknown) of
-        unknown ->
-            %% e.g. 23.2.7.1-emqx-2-x86_64-unknown-linux-gnu-64
-            OtpVsn = emqx_vm:get_otp_version(),
-            Bool = (match =:= re:run(OtpVsn, "emqx", [{capture, none}])),
-            _ = persistent_term:put({?MODULE, ipv6_probe_supported}, Bool),
-            ipv6_probe(Bool, Opts);
-        Bool ->
-            ipv6_probe(Bool, Opts)
-    end.
+    Bool = try gen_tcp:ipv6_probe()
+           catch _ : _ -> false end,
+    ipv6_probe(Bool, Opts).
 
 ipv6_probe(false, Opts) -> Opts;
 ipv6_probe(true, Opts) -> [{ipv6_probe, true} | Opts].

From 2251159c4fe613c10c416a5bd22702da719c0fc4 Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Tue, 16 Nov 2021 17:51:38 +0800
Subject: [PATCH 13/16] build: pin otp version to 23.3.4.9-3 and builder
 version to 4.4-2

---
 .github/workflows/build_packages.yaml          | 12 ++++++------
 .github/workflows/build_slim_packages.yaml     |  6 +++---
 .github/workflows/check_deps_integrity.yaml    |  2 +-
 .github/workflows/run_acl_migration_tests.yaml |  2 +-
 .github/workflows/run_automate_tests.yaml      |  2 +-
 .github/workflows/run_fvt_tests.yaml           | 12 ++++++------
 .github/workflows/run_test_cases.yaml          |  4 ++--
 build                                          |  2 +-
 deploy/docker/Dockerfile                       |  2 +-
 9 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index f468c8c04..83c10a4c9 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -11,7 +11,7 @@ on:
 jobs:
   prepare:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
 
     outputs:
       profiles: ${{ steps.set_profile.outputs.profiles}}
@@ -134,7 +134,7 @@ jobs:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         erl_otp:
-          - 23.3.4.9-2
+          - 23.3.4.9-3
         exclude:
           - profile: emqx-edge
 
@@ -213,7 +213,7 @@ jobs:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         otp:
-          - 23.3.4.9-2
+          - 23.3.4.9-3
         arch:
           - amd64
           - arm64
@@ -301,7 +301,7 @@ jobs:
         -v $(pwd):/emqx \
         --workdir /emqx \
         --platform linux/$ARCH \
-        ghcr.io/emqx/emqx-builder/4.4-1:$OTP-$SYSTEM \
+        ghcr.io/emqx/emqx-builder/4.4-2:$OTP-$SYSTEM \
         bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \
                    make $PROFILE-pkg || cat rebar3.crashdump; \
                    EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh"
@@ -331,7 +331,7 @@ jobs:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         otp:
-          - 23.3.4.9-2
+          - 23.3.4.9-3
         arch:
           - [amd64, x86_64]
           - [arm64v8, aarch64]
@@ -382,7 +382,7 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:${{ matrix.otp }}-alpine3.14
+          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:${{ matrix.otp }}-alpine3.14
           RUN_FROM=alpine:3.14
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile
diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index 5ee2a63dd..5888dff8b 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -15,12 +15,12 @@ jobs:
     strategy:
       matrix:
         erl_otp:
-        - 23.3.4.9-2
+        - 23.3.4.9-3
         os:
         - ubuntu20.04
         - centos7
 
-    container: ghcr.io/emqx/emqx-builder/4.4-1:${{ matrix.erl_otp }}-${{ matrix.os }}
+    container: ghcr.io/emqx/emqx-builder/4.4-2:${{ matrix.erl_otp }}-${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v1
@@ -58,7 +58,7 @@ jobs:
     strategy:
       matrix:
         erl_otp:
-        - 23.3.4.9-2
+        - 23.3.4.9-3
 
     steps:
     - uses: actions/checkout@v1
diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml
index 5ebd2f69d..b8c4a5c18 100644
--- a/.github/workflows/check_deps_integrity.yaml
+++ b/.github/workflows/check_deps_integrity.yaml
@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
 
     steps:
       - uses: actions/checkout@v2
diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml
index 4cd2a914b..5f452194c 100644
--- a/.github/workflows/run_acl_migration_tests.yaml
+++ b/.github/workflows/run_acl_migration_tests.yaml
@@ -5,7 +5,7 @@ on: workflow_dispatch
 jobs:
     test:
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
         strategy:
             fail-fast: true
         env:
diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml
index e4680be78..ca7a84612 100644
--- a/.github/workflows/run_automate_tests.yaml
+++ b/.github/workflows/run_automate_tests.yaml
@@ -33,7 +33,7 @@ jobs:
         fi
     - name: build docker image
       env:
-        OTP_VSN: 23.3.4.9-2
+        OTP_VSN: 23.3.4.9-3
       run: |
         make ${{ steps.prepare.outputs.imgname }}-docker
         docker save emqx/${{ steps.prepare.outputs.imgname }}:${{ steps.prepare.outputs.version }} -o image.tar.gz
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index 2aece7312..a16586bd9 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -31,7 +31,7 @@ jobs:
             fi
         - name: make emqx image
           env:
-            OTP_VSN: 23.3.4.9-2
+            OTP_VSN: 23.3.4.9-3
           run: make ${PROFILE}-docker
         - name: run emqx
           timeout-minutes: 5
@@ -86,7 +86,7 @@ jobs:
             fi
         - name: make emqx image
           env:
-            OTP_VSN: 23.3.4.9-2
+            OTP_VSN: 23.3.4.9-3
           run: make ${PROFILE}-docker
         - name: install k3s
           env:
@@ -183,7 +183,7 @@ jobs:
 
     relup_test_plan:
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
         outputs:
           profile: ${{ steps.profile-and-versions.outputs.profile }}
           vsn: ${{ steps.profile-and-versions.outputs.vsn }}
@@ -232,7 +232,7 @@ jobs:
     relup_test_build:
         needs: relup_test_plan
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
         defaults:
           run:
             shell: bash
@@ -319,8 +319,8 @@ jobs:
             --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
             --var VSN="$VSN" \
             --var OLD_VSN="$OLD_VSN" \
-            --var FROM_OTP_VSN="23.3.4.9-2" \
-            --var TO_OTP_VSN="23.3.4.9-2" \
+            --var FROM_OTP_VSN="23.3.4.9-3" \
+            --var TO_OTP_VSN="23.3.4.9-3" \
             emqx_built/.ci/fvt_tests/relup.lux
         - uses: actions/upload-artifact@v2
           name: Save debug data
diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 9d002138e..2f5570e28 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -10,7 +10,7 @@ on:
 jobs:
     run_static_analysis:
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
 
         steps:
         - uses: actions/checkout@v2
@@ -27,7 +27,7 @@ jobs:
 
     run_proper_test:
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
 
         steps:
         - uses: actions/checkout@v2
diff --git a/build b/build
index f29dbc985..24e79dda3 100755
--- a/build
+++ b/build
@@ -127,7 +127,7 @@ make_zip() {
 make_docker() {
     EMQX_RUNNER_IMAGE='alpine:3.14'
     EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')"
-    EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}"
+    EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-2:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}"
 
     ## Build Docker image
     echo "DOCKER BUILD: Build Docker image."
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index b48011bfd..88f80bdf1 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -1,4 +1,4 @@
-ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-alpine3.14
+ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14
 ARG RUN_FROM=alpine:3.14
 FROM ${BUILD_FROM} AS builder
 

From 39c564c072d7934c29d543dc7a72c510a80ded71 Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Tue, 16 Nov 2021 17:55:27 +0800
Subject: [PATCH 14/16] ci: rename windows packae

---
 .github/workflows/build_packages.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 83c10a4c9..46133e95b 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -65,6 +65,8 @@ jobs:
     strategy:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+        otp:
+          - 23.2
         exclude:
           - profile: emqx-edge
 
@@ -79,7 +81,7 @@ jobs:
     - uses: gleam-lang/setup-erlang@v1.1.0
       id: install_erlang
       with:
-        otp-version: 23.2
+        otp-version: ${{ matrix.otp }}
     - name: build
       env:
         PYTHON: python
@@ -90,10 +92,10 @@ jobs:
         $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
         if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
           $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
-          $pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip"
+          $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip"
           }
         else {
-          $pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip"
+          $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip"
           }
         cd source
         ## We do not build/release bcrypt for windows package

From 284d1223728e43f796bdcfb6c04cd076f4993029 Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Tue, 16 Nov 2021 18:33:21 +0800
Subject: [PATCH 15/16] ci: pin otp version to 23.3.4.9-3 and builder version
 to 4.4-2

---
 .ci/build_packages/Dockerfile               | 2 +-
 .ci/docker-compose-file/docker-compose.yaml | 2 +-
 .ci/fvt_tests/local_relup_test_run.sh       | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile
index 0a7c7496d..1063a2c13 100644
--- a/.ci/build_packages/Dockerfile
+++ b/.ci/build_packages/Dockerfile
@@ -1,4 +1,4 @@
-ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
+ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
 FROM ${BUILD_FROM}
 
 ARG EMQX_NAME=emqx
diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml
index 0415ac142..8bd6455e9 100644
--- a/.ci/docker-compose-file/docker-compose.yaml
+++ b/.ci/docker-compose-file/docker-compose.yaml
@@ -3,7 +3,7 @@ version: '3.9'
 services:
   erlang:
     container_name: erlang
-    image: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04
+    image: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
     env_file:
       - conf.env
     environment:
diff --git a/.ci/fvt_tests/local_relup_test_run.sh b/.ci/fvt_tests/local_relup_test_run.sh
index 576ed6d02..8a563a217 100755
--- a/.ci/fvt_tests/local_relup_test_run.sh
+++ b/.ci/fvt_tests/local_relup_test_run.sh
@@ -15,8 +15,8 @@ PROFILE="$1"
 VSN="$2"
 OLD_VSN="$3"
 PACKAGE_PATH="$4"
-FROM_OTP_VSN="${5:-23.3.4.9-2}"
-TO_OTP_VSN="${6:-23.3.4.9-2}"
+FROM_OTP_VSN="${5:-23.3.4.9-3}"
+TO_OTP_VSN="${6:-23.3.4.9-3}"
 
 TEMPDIR=$(mktemp -d)
 trap '{ rm -rf -- "$TEMPDIR"; }' EXIT

From db802ad04ff347d5fafc3fe3691fd6094449b549 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Tue, 16 Nov 2021 11:35:13 +0100
Subject: [PATCH 16/16] chore: update toos-versions to pin 23.3.4.9-3

---
 .tool-versions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.tool-versions b/.tool-versions
index a42915120..757309f18 100644
--- a/.tool-versions
+++ b/.tool-versions
@@ -1 +1 @@
-erlang 23.3.4.9-2
+erlang 23.3.4.9-3