build(relup): add new script for download relup base packages

.github/workflows/build_packages.yaml

@@ -15,7 +15,6 @@ jobs:
 
     outputs:
       profiles: ${{ steps.set_profile.outputs.profiles}}
-      old_vsns: ${{ steps.set_profile.outputs.old_vsns}}
 
     steps:
       - uses: actions/checkout@v2
@@ -28,12 +27,8 @@ jobs:
         run: |
           cd source
           if make emqx-ee --dry-run > /dev/null 2>&1; then
-            old_vsns="$(./scripts/relup-base-vsns.sh enterprise | xargs)"
-            echo "::set-output name=old_vsns::$old_vsns"
             echo "::set-output name=profiles::[\"emqx-ee\"]"
           else
-            old_vsns="$(./scripts/relup-base-vsns.sh community | xargs)"
-            echo "::set-output name=old_vsns::$old_vsns"
             echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
           fi
       - name: get_all_deps
@@ -216,6 +211,7 @@ jobs:
     needs: prepare
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         arch:
@@ -269,32 +265,6 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - name: downloads old emqx zip packages
-      env:
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch }}
-        SYSTEM: ${{ matrix.os }}
-        OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }}
-      run: |
-        set -e -x -u
-        broker=$PROFILE
-        if [ $PROFILE = "emqx" ];then
-            broker="emqx-ce"
-        fi
-        if [ ! -z "$(echo $SYSTEM | grep -oE 'raspbian')" ]; then
-            export ARCH="arm"
-        fi
-
-        mkdir -p source/_upgrade_base
-        cd source/_upgrade_base
-        old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
-        for tag in ${old_vsns[@]}; do
-          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256
-            echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1
-          fi
-        done
     - name: build emqx packages
       env:
         ERL_OTP: erl23.2.7.2-emqx-3
@@ -342,6 +312,7 @@ jobs:
     needs: prepare
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         registry:
@@ -439,6 +410,7 @@ jobs:
     needs: [prepare, mac, linux, docker]
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
 

.github/workflows/run_fvt_tests.yaml

@@ -239,6 +239,7 @@ jobs:
           name: Checkout
           with:
             path: emqx
+            fetch-depth: 0
         - name: Prepare credentials
           run: |
             if [ "$PROFILE" = "emqx-ee" ]; then
@@ -246,15 +247,6 @@ jobs:
               git config --global credential.helper store
               echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token
             fi
-        - name: Download bases
-          run: |
-            set -e -x -u
-            mkdir -p emqx/_upgrade_base
-            cd emqx/_upgrade_base
-            old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
-            for old_vsn in ${old_vsns[@]}; do
-              wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip
-            done
         - name: Build emqx
           run: make -C emqx ${PROFILE}-zip
         - uses: actions/upload-artifact@v2

Makefile

@@ -3,6 +3,7 @@ REBAR_VERSION = 3.14.3-emqx-8
 REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
+export EMQX_RELUP ?= true
 export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.2.7.2-emqx-3-alpine
 export EMQX_DEFAULT_RUNNER = alpine:3.12
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
@@ -125,10 +126,19 @@ COMMON_DEPS := $(REBAR) get-dashboard $(CONF_SEGS)
 $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS)
 	@$(BUILD) $(subst -rel,,$(@)) rel
 
+## download relup base packages
+.PHONY: $(REL_PROFILES:%=%-relup-downloads)
+define download-relup-packages
+$1-relup-downloads:
+	@if [ "$${EMQX_RELUP}" = "true" ]; then $(CURDIR)/scripts/relup-base-packages.sh $1; fi
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call download-relup-packages,$(zt))))
+
 ## relup target is to create relup instructions
 .PHONY: $(REL_PROFILES:%=%-relup)
 define gen-relup-target
-$1-relup: $(COMMON_DEPS)
+$1-relup: $1-relup-downloads $(COMMON_DEPS)
 	@$(BUILD) $1 relup
 endef
 ALL_ZIPS = $(REL_PROFILES)

build

@@ -4,6 +4,9 @@
 # arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg
 # arg2: artifact, e.g. rel | relup | zip | pkg
 
+if [[ -n "$DEBUG" ]]; then
+    set -x
+fi
 set -euo pipefail
 
 PROFILE="$1"
@@ -71,8 +74,8 @@ make_relup() {
                 tmp_dir="$(mktemp -d -t emqx.XXXXXXX)"
                 unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir"
                 unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir"
-                cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir"
-                cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir"
+                cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" || true
+                cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" || true
                 rm -rf "$tmp_dir"
             fi
             releases+=( "$base_vsn" )

deploy/docker/Dockerfile

@@ -23,6 +23,8 @@ COPY . /emqx
 ARG PKG_VSN
 ARG EMQX_NAME=emqx
 
+ENV EMQX_RELUP=false
+
 RUN cd /emqx \
     && rm -rf _build/$EMQX_NAME/lib \
     && make $EMQX_NAME

scripts/relup-base-packages.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+if [[ -n "$DEBUG" ]]; then
+    set -x
+fi
+set -euo pipefail
+
+PROFILE="${1}"
+if [ "$PROFILE" = "" ]; then
+    $PROFILE="emqx"
+fi
+
+case $PROFILE in
+    "emqx")
+        BROKER="broker"
+        ;;
+    "emqx-ee")
+        BROKER="enterprise"
+        ;;
+    "emqx-edge")
+        BROKER="edge"
+        ;;
+esac
+
+SYSTEM="$(./scripts/get-distro.sh)"
+
+ARCH="$(uname -m)"
+case "$ARCH" in
+    x86_64)
+        ARCH='amd64'
+        ;;
+    aarch64)
+        ARCH='arm64'
+        ;;
+    arm*)
+        ARCH=arm
+        ;;
+esac
+
+SHASUM="sha256sum"
+if [ "$SYSTEM" = "macos" ]; then
+    SHASUM="shasum -a 256"
+fi
+
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+
+mkdir -p _upgrade_base
+pushd _upgrade_base
+
+for tag in $(../scripts/relup-base-vsns.sh community | xargs echo -n); do
+    if [ ! -f "$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" ] \
+    && [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://www.emqx.com/downloads/$BROKER/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
+        wget --no-verbose https://www.emqx.com/downloads/$BROKER/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip
+        if [ "$SYSTEM" != "centos6" ]; then
+            wget --no-verbose https://www.emqx.com/downloads/$BROKER/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256
+            ## https://askubuntu.com/questions/1202208/checking-sha256-checksum
+            echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256)  $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | $SHASUM -c || exit 1
+        fi
+    fi
+done 
+
+popd

scripts/relup-base-vsns.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+
 ## This script prints the relup upgrade base versions
 ## for the given EMQ X edition (specified as first arg)
 ##