Merge pull request #7028 from Rory-Z/build/add-download-relup

build(relup): add new script for download relup base packages
2022-02-17 06:28:20 +01:00 · 2022-02-17 06:28:20 +01:00 · ee544c8298
parent 70ce5cbab3 093e7c3282
commit ee544c8298
7 changed files with 95 additions and 83 deletions
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@ -25,8 +25,6 @@ jobs:
    container: "ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-ubuntu20.04"

    outputs:
-      ce_old_vsns: ${{ steps.find_old_versons.outputs.ce_old_vsns }}
-      ee_old_vsns: ${{ steps.find_old_versons.outputs.ee_old_vsns }}
      DEP_ROCKSDB_REF: ${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
      DEP_QUICER_REF: ${{ steps.deps-refs.outputs.DEP_QUICER_REF }}

@ -43,16 +41,6 @@ jobs:
        run: |
          bash -x scripts/get-dep-refs.sh
          make clean-all
-
-      - name: find old versions
-        id: find_old_versons
-        shell: bash
-        working-directory: source
-        run: |
-          ce_old_vsns="$(./scripts/relup-base-vsns.sh community  | xargs)"
-          ee_old_vsns="$(./scripts/relup-base-vsns.sh enterprise | xargs)"
-          echo "::set-output name=ce_old_vsns::${ce_old_vsns}"
-          echo "::set-output name=ee_old_vsns::${ee_old_vsns}"
      - name: get_all_deps
        run: |
          make -C source deps-all
@ -332,47 +320,6 @@ jobs:
          source/_build/default/lib/quicer/
          source/deps/quicer/
        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ needs.prepare.outputs.DEP_QUICER_REF }}
-    - name: download old emqx tgz packages
-      env:
-        OTP_VSN: ${{ matrix.otp }}
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch }}
-        SYSTEM: ${{ matrix.os }}
-        CE_OLD_VSNS: ${{ needs.prepare.outputs.ce_old_vsns }}
-        EE_OLD_VSNS: ${{ needs.prepare.outputs.ee_old_vsns }}
-      working-directory: source
-      run: |
-        set -e -x -u
-        if [ $PROFILE = 'emqx' ]; then
-            s3dir='emqx-ce'
-            OLD_VSNS="$CE_OLD_VSNS"
-        elif [ $PROFILE = 'emqx-enterprise' ]; then
-            s3dir='emqx-ee'
-            OLD_VSNS="$EE_OLD_VSNS"
-        elif [ $PROFILE = 'emqx-edge' ]; then
-            s3dir='emqx-edge'
-            OLD_VSNS="$CE_OLD_VSNS"
-        else
-            echo "unknown profile $PROFILE"
-            exit 1
-        fi
-        mkdir -p _upgrade_base
-        cd _upgrade_base
-        old_vsns=$(echo "$OLD_VSNS" | tr ' ' ' ')
-
-        # workaround for bash empty array expanding issue in different bash versions
-        if [ -n "$old_vsns" ]; then
-        old_vsns=($old_vsns)
-        for tag in ${old_vsns[@]}; do
-          package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
-          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$s3dir/$tag/$package_name.tar.gz) | grep -oE "^[23]+")" ]; then
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$s3dir/$tag/$package_name.tar.gz
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$s3dir/$tag/$package_name.tar.gz.sha256
-            echo "$(cat $package_name.tar.gz.sha256) $package_name.tar.gz" | sha256sum -c || exit 1
-          fi
-        done
-        fi
-
    - name: build emqx packages
      working-directory: source
      env:
@ -700,6 +647,7 @@ jobs:
    needs: [prepare, mac, linux, docker]

    strategy:
+      fail-fast: false
      matrix:
        profile:
          - emqx-edge
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@ -68,36 +68,9 @@ jobs:
        repository: ${{ github.repository }}
        path: emqx
        fetch-depth: 0
-    - name: get version
-      run: |
-        set -e -x -u
-        cd emqx
-        export PROFILE=${{ matrix.profile }}
-        export OTP_VSN=${{ matrix.otp }}
-        echo "PROFILE=$PROFILE" >> $GITHUB_ENV
-        echo "OTP_VSN=$OTP_VSN" >> $GITHUB_ENV
-        if [ $PROFILE = "emqx" ];then
-            broker="emqx-ce"
-        else
-            broker="emqx-enterprise"
-        fi
-        echo "BROKER=$broker" >> $GITHUB_ENV

-        vsn="$(./pkg-vsn.sh $PROFILE)"
-        echo "VSN=$vsn" >> $GITHUB_ENV
-
-        pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
-        old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
-        echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
-    - name: download emqx
-      run: |
-        set -e -x -u
-        mkdir -p emqx/_upgrade_base
-        cd emqx/_upgrade_base
-        old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
-        for old_vsn in ${old_vsns[@]}; do
-          wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-${{ matrix.os }}-${{ matrix.arch }}.tar.gz
-        done
+    - name: Get old vsn
+      run: echo "OLD_VSNS=$(emqx/scripts/relup-base-vsns.sh ${{ matrix.profile }} | xargs echo -n)" >> $GITHUB_ENV

    - name: Get deps git refs for cache
      id: deps-refs
@ -117,6 +90,8 @@ jobs:
        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}

    - name: build emqx
+      env:
+        PROFILE: ${{ matrix.profile }}
      run: make -C emqx ${PROFILE}-tgz
    - name: build emqtt-bench
      run: make -C emqtt-bench
@ -129,6 +104,8 @@ jobs:
        make
        make install
    - name: run relup test
+      env:
+        PROFILE: ${{ matrix.profile }}
      timeout-minutes: 20
      run: |
        set -e -x -u
--- a/12
+++ b/12
@ -3,6 +3,7 @@ REBAR_VERSION = 3.16.1-emqx-1
 REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
+export EMQX_RELUP ?= true
 export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-5:1.13.2-24.1.5-4-alpine3.14
 export EMQX_DEFAULT_RUNNER = alpine:3.14
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
@ -162,10 +163,19 @@ dialyzer: $(REBAR)
 $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS)
 	@$(BUILD) $(subst -rel,,$(@)) rel

+## download relup base packages
+.PHONY: $(REL_PROFILES:%=%-relup-downloads)
+define download-relup-packages
+$1-relup-downloads:
+	@if [ "$${EMQX_RELUP}" = "true" ]; then $(CURDIR)/scripts/relup-base-packages.sh $1; fi
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call download-relup-packages,$(zt))))
+
 ## relup target is to create relup instructions
 .PHONY: $(REL_PROFILES:%=%-relup)
 define gen-relup-target
-$1-relup: $(COMMON_DEPS)
+$1-relup: $1-relup-downloads $(COMMON_DEPS)
 	@$(BUILD) $1 relup
 endef
 ALL_TGZS = $(REL_PROFILES)
--- a/3
+++ b/3
@ -4,6 +4,9 @@
 # arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg
 # arg2: artifact, e.g. rel | relup | tgz | pkg

+if [[ -n "$DEBUG" ]]; then
+    set -x
+fi
 set -euo pipefail

 DEBUG="${DEBUG:-0}"
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@ -21,6 +21,7 @@ RUN apk add --no-cache \
 COPY . /emqx

 ARG EMQX_NAME=emqx
+ENV EMQX_RELUP=false

 RUN export PROFILE="$EMQX_NAME" \
    && export EMQX_NAME=${EMQX_NAME%%-elixir} \
--- a/scripts/relup-base-packages.sh
+++ b/scripts/relup-base-packages.sh
@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+## This script helps to download relup base version packages
+
+if [[ -n "$DEBUG" ]]; then set -x; fi
+set -euo pipefail
+
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+
+PROFILE="${1:-}"
+if [ "$PROFILE" = "" ]; then
+    PROFILE="emqx"
+fi
+
+OTP_VSN=${OTP_VSN:-$(scripts/get-otp-vsn.sh)}
+
+case $PROFILE in
+    "emqx")
+        DIR='broker'
+        EDITION='community'
+        ;;
+    "emqx-ee")
+        DIR='enterprise'
+        EDITION='enterprise'
+        ;;
+    "emqx-edge")
+        DIR='edge'
+        EDITION='edge'
+        ;;
+esac
+
+SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}"
+
+ARCH="${ARCH:-$(uname -m)}"
+case "$ARCH" in
+    x86_64)
+        ARCH='amd64'
+        ;;
+    aarch64)
+        ARCH='arm64'
+        ;;
+    arm*)
+        ARCH=arm
+        ;;
+esac
+
+SHASUM="sha256sum"
+if [ "$SYSTEM" = "macos" ]; then
+    SHASUM="shasum -a 256"
+fi
+
+mkdir -p _upgrade_base
+pushd _upgrade_base
+
+for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do
+    filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.tar.gz"
+    url="https://www.emqx.com/downloads/$DIR/$tag/$filename"
+    echo "downloading ${filename} ..."
+    ## if the file does not exist (not downloaded yet)
+    ## and there is such a package to downlaod
+    if [ ! -f "$filename" ] && curl -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then
+        curl -L -o "${filename}" "${url}"
+        curl -L -o "${filename}.sha256" "${url}.sha256"
+        ## https://askubuntu.com/questions/1202208/checking-sha256-checksum
+        echo "$(cat "${filename}.sha256")  ${filename}" | $SHASUM -c || exit 1
+    fi
+done
+
+popd
--- a/scripts/relup-base-vsns.sh
+++ b/scripts/relup-base-vsns.sh
@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 set -euo pipefail

+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+
 ## This script prints the relup upgrade base versions
 ## for the given EMQX edition (specified as first arg)
 ##