Merge pull request #7028 from Rory-Z/build/add-download-relup

build(relup): add new script for download relup base packages

.github/workflows/build_packages.yaml

@@ -25,8 +25,6 @@ jobs:
     container: "ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-ubuntu20.04"
 
     outputs:
-      ce_old_vsns: ${{ steps.find_old_versons.outputs.ce_old_vsns }}
-      ee_old_vsns: ${{ steps.find_old_versons.outputs.ee_old_vsns }}
       DEP_ROCKSDB_REF: ${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
       DEP_QUICER_REF: ${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
 
@@ -43,16 +41,6 @@ jobs:
         run: |
           bash -x scripts/get-dep-refs.sh
           make clean-all
-
-      - name: find old versions
-        id: find_old_versons
-        shell: bash
-        working-directory: source
-        run: |
-          ce_old_vsns="$(./scripts/relup-base-vsns.sh community  | xargs)"
-          ee_old_vsns="$(./scripts/relup-base-vsns.sh enterprise | xargs)"
-          echo "::set-output name=ce_old_vsns::${ce_old_vsns}"
-          echo "::set-output name=ee_old_vsns::${ee_old_vsns}"
       - name: get_all_deps
         run: |
           make -C source deps-all
@@ -332,47 +320,6 @@ jobs:
           source/_build/default/lib/quicer/
           source/deps/quicer/
         key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ needs.prepare.outputs.DEP_QUICER_REF }}
-    - name: download old emqx tgz packages
-      env:
-        OTP_VSN: ${{ matrix.otp }}
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch }}
-        SYSTEM: ${{ matrix.os }}
-        CE_OLD_VSNS: ${{ needs.prepare.outputs.ce_old_vsns }}
-        EE_OLD_VSNS: ${{ needs.prepare.outputs.ee_old_vsns }}
-      working-directory: source
-      run: |
-        set -e -x -u
-        if [ $PROFILE = 'emqx' ]; then
-            s3dir='emqx-ce'
-            OLD_VSNS="$CE_OLD_VSNS"
-        elif [ $PROFILE = 'emqx-enterprise' ]; then
-            s3dir='emqx-ee'
-            OLD_VSNS="$EE_OLD_VSNS"
-        elif [ $PROFILE = 'emqx-edge' ]; then
-            s3dir='emqx-edge'
-            OLD_VSNS="$CE_OLD_VSNS"
-        else
-            echo "unknown profile $PROFILE"
-            exit 1
-        fi
-        mkdir -p _upgrade_base
-        cd _upgrade_base
-        old_vsns=$(echo "$OLD_VSNS" | tr ' ' ' ')
-
-        # workaround for bash empty array expanding issue in different bash versions
-        if [ -n "$old_vsns" ]; then
-        old_vsns=($old_vsns)
-        for tag in ${old_vsns[@]}; do
-          package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
-          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$s3dir/$tag/$package_name.tar.gz) | grep -oE "^[23]+")" ]; then
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$s3dir/$tag/$package_name.tar.gz
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$s3dir/$tag/$package_name.tar.gz.sha256
-            echo "$(cat $package_name.tar.gz.sha256) $package_name.tar.gz" | sha256sum -c || exit 1
-          fi
-        done
-        fi
-
     - name: build emqx packages
       working-directory: source
       env:
@@ -700,6 +647,7 @@ jobs:
     needs: [prepare, mac, linux, docker]
 
     strategy:
+      fail-fast: false
       matrix:
         profile:
           - emqx-edge

.github/workflows/run_relup_tests.yaml

@@ -68,36 +68,9 @@ jobs:
         repository: ${{ github.repository }}
         path: emqx
         fetch-depth: 0
-    - name: get version
-      run: |
-        set -e -x -u
-        cd emqx
-        export PROFILE=${{ matrix.profile }}
-        export OTP_VSN=${{ matrix.otp }}
-        echo "PROFILE=$PROFILE" >> $GITHUB_ENV
-        echo "OTP_VSN=$OTP_VSN" >> $GITHUB_ENV
-        if [ $PROFILE = "emqx" ];then
-            broker="emqx-ce"
-        else
-            broker="emqx-enterprise"
-        fi
-        echo "BROKER=$broker" >> $GITHUB_ENV
 
-        vsn="$(./pkg-vsn.sh $PROFILE)"
+    - name: Get old vsn
-        echo "VSN=$vsn" >> $GITHUB_ENV
+      run: echo "OLD_VSNS=$(emqx/scripts/relup-base-vsns.sh ${{ matrix.profile }} | xargs echo -n)" >> $GITHUB_ENV
-
-        pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
-        old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
-        echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
-    - name: download emqx
-      run: |
-        set -e -x -u
-        mkdir -p emqx/_upgrade_base
-        cd emqx/_upgrade_base
-        old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
-        for old_vsn in ${old_vsns[@]}; do
-          wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-${{ matrix.os }}-${{ matrix.arch }}.tar.gz
-        done
 
     - name: Get deps git refs for cache
       id: deps-refs
@@ -117,6 +90,8 @@ jobs:
         key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
 
     - name: build emqx
+      env:
+        PROFILE: ${{ matrix.profile }}
       run: make -C emqx ${PROFILE}-tgz
     - name: build emqtt-bench
       run: make -C emqtt-bench
@@ -129,6 +104,8 @@ jobs:
         make
         make install
     - name: run relup test
+      env:
+        PROFILE: ${{ matrix.profile }}
       timeout-minutes: 20
       run: |
         set -e -x -u

Makefile

@@ -3,6 +3,7 @@ REBAR_VERSION = 3.16.1-emqx-1
 REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
+export EMQX_RELUP ?= true
 export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-alpine3.14
 export EMQX_DEFAULT_RUNNER = alpine:3.14
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
@@ -162,10 +163,19 @@ dialyzer: $(REBAR)
 $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS)
 	@$(BUILD) $(subst -rel,,$(@)) rel
 
+## download relup base packages
+.PHONY: $(REL_PROFILES:%=%-relup-downloads)
+define download-relup-packages
+$1-relup-downloads:
+	@if [ "$${EMQX_RELUP}" = "true" ]; then $(CURDIR)/scripts/relup-base-packages.sh $1; fi
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call download-relup-packages,$(zt))))
+
 ## relup target is to create relup instructions
 .PHONY: $(REL_PROFILES:%=%-relup)
 define gen-relup-target
-$1-relup: $(COMMON_DEPS)
+$1-relup: $1-relup-downloads $(COMMON_DEPS)
 	@$(BUILD) $1 relup
 endef
 ALL_TGZS = $(REL_PROFILES)

build

@@ -4,6 +4,9 @@
 # arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg
 # arg2: artifact, e.g. rel | relup | tgz | pkg
 
+if [[ -n "$DEBUG" ]]; then
+    set -x
+fi
 set -euo pipefail
 
 DEBUG="${DEBUG:-0}"

deploy/docker/Dockerfile

@@ -21,6 +21,7 @@ RUN apk add --no-cache \
 COPY . /emqx
 
 ARG EMQX_NAME=emqx
+ENV EMQX_RELUP=false
 
 RUN export PROFILE="$EMQX_NAME" \
     && export EMQX_NAME=${EMQX_NAME%%-elixir} \

scripts/relup-base-packages.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+## This script helps to download relup base version packages
+
+if [[ -n "$DEBUG" ]]; then set -x; fi
+set -euo pipefail
+
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+
+PROFILE="${1:-}"
+if [ "$PROFILE" = "" ]; then
+    PROFILE="emqx"
+fi
+
+OTP_VSN=${OTP_VSN:-$(scripts/get-otp-vsn.sh)}
+
+case $PROFILE in
+    "emqx")
+        DIR='broker'
+        EDITION='community'
+        ;;
+    "emqx-ee")
+        DIR='enterprise'
+        EDITION='enterprise'
+        ;;
+    "emqx-edge")
+        DIR='edge'
+        EDITION='edge'
+        ;;
+esac
+
+SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}"
+
+ARCH="${ARCH:-$(uname -m)}"
+case "$ARCH" in
+    x86_64)
+        ARCH='amd64'
+        ;;
+    aarch64)
+        ARCH='arm64'
+        ;;
+    arm*)
+        ARCH=arm
+        ;;
+esac
+
+SHASUM="sha256sum"
+if [ "$SYSTEM" = "macos" ]; then
+    SHASUM="shasum -a 256"
+fi
+
+mkdir -p _upgrade_base
+pushd _upgrade_base
+
+for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do
+    filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.tar.gz"
+    url="https://www.emqx.com/downloads/$DIR/$tag/$filename"
+    echo "downloading ${filename} ..."
+    ## if the file does not exist (not downloaded yet)
+    ## and there is such a package to downlaod
+    if [ ! -f "$filename" ] && curl -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then
+        curl -L -o "${filename}" "${url}"
+        curl -L -o "${filename}.sha256" "${url}.sha256"
+        ## https://askubuntu.com/questions/1202208/checking-sha256-checksum
+        echo "$(cat "${filename}.sha256")  ${filename}" | $SHASUM -c || exit 1
+    fi
+done
+
+popd

scripts/relup-base-vsns.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+
 ## This script prints the relup upgrade base versions
 ## for the given EMQX edition (specified as first arg)
 ##