build: get full version from pkg-full-vsn.sh

this is due to the mapping of :
centos7 -> el7
and rockylinux8 -> el8

with this fix, there is no need to repeat such mapping
in many differennt places

.github/workflows/spelling.yml

@@ -23,7 +23,7 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/download-artifact@v2
       with:
-        name:  ${{ matrix.profile }}-${{ mqtrix.otp }}-${{ matrix.os }}
+        name:  ${{ matrix.profile }}-${{ matrix.otp }}-${{ matrix.os }}
         workflow: build_slim_packages.yaml
         path: .
     - name: Run spellcheck

Makefile

@@ -10,7 +10,6 @@ export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
 export EMQX_DASHBOARD_VERSION ?= v0.20.0
 export DOCKERFILE := deploy/docker/Dockerfile
-export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
 ifeq ($(OS),Windows_NT)
 	export REBAR_COLOR=none
 	FIND=/usr/bin/find
@@ -214,18 +213,6 @@ endef
 ALL_DOCKERS = $(REL_PROFILES) $(REL_PROFILES:%=%-elixir)
 $(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt))))
 
-## emqx-docker-testing
-## emqx-enterprise-docker-testing
-## is to directly copy a extracted tgz-package to a
-## base image such as ubuntu20.04. Mostly for testing
-.PHONY: $(REL_PROFILES:%=%-docker-testing)
-define gen-docker-target-testing
-$1-docker-testing: $(COMMON_DEPS)
-	@$(BUILD) $1 docker-testing
-endef
-ALL_TGZS = $(REL_PROFILES)
-$(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt))))
-
 .PHONY:
 conf-segs:
 	@scripts/merge-config.escript

build

@@ -95,13 +95,15 @@ make_relup() {
     local rel_dir="_build/$PROFILE/rel/emqx"
     mkdir -p "${rel_dir}/lib"
     mkdir -p "${rel_dir}/releases"
+    local name_pattern
+    name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh "$PROFILE" 'vsn_matcher')"
     local releases=()
     while read -r tgzfile ; do
         local base_vsn
         base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta)\.[0-9])?(-[0-9a-f]{8})?" | head -1)"
         tar -C "$rel_dir" -zxf ---keep-old-files "$tgzfile" emqx/releases emqx/lib
         releases+=( "$base_vsn" )
-    done < <("$FIND" _upgrade_base -maxdepth 1 -name "$PROFILE*${SYSTEM}-${ARCH}.tar.gz" -type f)
+    done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.tar.gz" -type f)
     if [ ${#releases[@]} -eq 0 ]; then
         log "No upgrade base found, relup ignored"
         return 0
@@ -141,15 +143,16 @@ make_tgz() {
       ELIXIR_MAKE_TAR=yes make_elixir_rel
 
       local relpath="_build/${PROFILE}"
-      target_name="${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
+      full_vsn="$(WITH_ELIXIR=yes ./scripts/pkg-full-vsn.sh "$PROFILE")"
     else
       # build the src_tarball again to ensure relup is included
       # elixir does not have relup yet.
       make_rel
 
       local relpath="_build/${PROFILE}/rel/emqx"
-      target_name="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
+      full_vsn="$(WITH_ELIXIR=no ./scripts/pkg-full-vsn.sh "$PROFILE")"
     fi
+    target_name="${PROFILE}-${full_vsn}.tar.gz"
     target="${pkgpath}/${target_name}"
 
     src_tarball="${relpath}/emqx-${PKG_VSN}.tar.gz"
@@ -201,46 +204,6 @@ make_docker() {
        -f "${DOCKERFILE}" .
 }
 
-## This function accepts any base docker image,
-## a emqx tgz-image, and a image tag (for the image to be built),
-## to build a docker image which runs EMQX
-##
-## Export below variables to quickly build an image
-##
-## Name               Default                         Example
-## ---------------------------------------------------------------------
-## EMQX_BASE_IMAGE    current os                      el:7
-## EMQX_TGZ_packages/<current-tgz-target>  /tmp/emqx-4.4.0-otp23.3.4.9-3-el7-amd64.tar.gz
-## EMQX_IMAGE_TAG     emqx/emqx:<current-vns-rel>     emqx/emqx:testing-tag
-##
-make_docker_testing() {
-    if [ -z "${EMQX_BASE_IMAGE:-}" ]; then
-        case "$SYSTEM" in
-            ubuntu20*)
-                EMQX_BASE_IMAGE="ubuntu:20.04"
-                ;;
-            *)
-                echo "Unsupported testing base image for $SYSTEM"
-                exit 1
-                ;;
-        esac
-    fi
-    EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}"
-    local default_tgz
-    default_tgz="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
-    local tgz="${EMQX_TGZ_PACKAGE:-$default_tgz}"
-    if [ ! -f "$tgz" ]; then
-        log "ERROR: $tgz not built?"
-        exit 1
-    fi
-    set -x
-    docker build \
-       --build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \
-       --build-arg EMQX_TGZ_PACKAGE="${tgz}" \
-       --tag "$EMQX_IMAGE_TAG" \
-       -f "${DOCKERFILE_TESTING}" .
-}
-
 function join {
   local IFS="$1"
   shift
@@ -310,9 +273,6 @@ case "$ARTIFACT" in
     docker)
         make_docker
         ;;
-    docker-testing)
-        make_docker_testing
-        ;;
     elixir)
         make_elixir_rel
         ;;

deploy/packages/deb/Makefile

@@ -6,16 +6,10 @@ BUILT := $(SRCDIR)/BUILT
 
 EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
 
-ifeq ($(IS_ELIXIR), yes)
-  ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN)
-else
-  ELIXIR_PKG_VSN :=
-endif
-
 TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx
 TAR_PKG     := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz
 SOURCE_PKG  := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture)
-TARGET_PKG  := $(EMQX_NAME)-$(PKG_VSN)$(ELIXIR_PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
+TARGET_PKG  := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh $(EMQX_NAME))
 
 .PHONY: all
 all: | $(BUILT)

deploy/packages/rpm/Makefile

@@ -16,16 +16,10 @@ endif
 
 EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
 
-ifeq ($(IS_ELIXIR), yes)
-  ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN)
-else
-  ELIXIR_PKG_VSN :=
-endif
-
 TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx
 TAR_PKG     := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz
 SOURCE_PKG  := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
-TARGET_PKG  := $(EMQX_NAME)-$(PKG_VSN)$(ELIXIR_PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
+TARGET_PKG  := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh $(EMQX_NAME))
 
 SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi)
 # Not $(PWD) as it does not work for make -C

scripts/buildx.sh

@@ -91,6 +91,9 @@ if [ -z "${PROFILE:-}" ]    ||
     exit 1
 fi
 
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+
 set -x
 
 if [ -z "${WITH_ELIXIR:-}" ]; then
@@ -112,16 +115,18 @@ cd "${CODE_PATH}"
 
 PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}"
 
-if [ "$WITH_ELIXIR" = "yes" ]
+if [ "$WITH_ELIXIR" = "yes" ]; then
-then
-  PKG_NAME="${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
   MAKE_TARGET="${PROFILE}-elixir-${PKGTYPE}"
 else
-  PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
   MAKE_TARGET="${PROFILE}-${PKGTYPE}"
 fi
 
-CMD_RUN="export EMQX_NAME=\"$PROFILE\"; make ${MAKE_TARGET} && ./scripts/pkg-tests.sh $PKG_NAME $PKGTYPE $ARCH"
+export WITH_ELIXIR
+export ELIXIR_VSN
+export PROFILE
+PKG_NAME="${PROFILE}-$(./scripts/pkg-full-vsn.sh "$PROFILE")"
+
+CMD_RUN="export EMQX_NAME=\"$PROFILE\"; make ${MAKE_TARGET} && ./scripts/pkg-tests.sh $PKG_NAME $PKGTYPE"
 
 if docker info; then
    docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}"

scripts/pkg-full-vsn.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+## This script print the package full vsn based on current build environment
+
+## Arg 1 is either 'vsn_exact' (default) or 'vsn_matcher'
+## when 'vsn_exact' is given, the version number is the output of pkg-vsn.sh
+## otherwise '*' is used for 'find' command to find old versions (as upgrade base)
+
+set -euo pipefail
+
+PROFILE="${1:-emqx}"
+VSN_MATCH="${2:-vsn_exact}"
+
+case "${VSN_MATCH}" in
+    vsn_exact)
+        PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}"
+        ;;
+    vsn_matcher)
+        PKG_VSN='*'
+        ;;
+    *)
+        echo "$0 ERROR: second arg must "
+        exit 1
+        ;;
+esac
+
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+
+OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}"
+SYSTEM="$(./scripts/get-distro.sh)"
+
+UNAME="$(uname -m)"
+case "$UNAME" in
+    x86_64)
+        ARCH='amd64'
+        ;;
+    aarch64)
+        ARCH='arm64'
+        ;;
+    arm*)
+        ARCH=arm
+        ;;
+esac
+
+if [[ "${WITH_ELIXIR:-}" == "yes" ]] || [[ "${IS_ELIXIR:-}" == "yes" ]] ; then
+    ELIXIR_VSN="${ELIXIR_VSN:-$(./scripts/get-elixir-vsn.sh)}"
+    FULL_VSN="${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
+else
+    FULL_VSN="${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
+fi
+
+echo "${FULL_VSN}"

scripts/pkg-tests.sh

@@ -26,6 +26,7 @@ export ARCH
 
 export DEBUG=1
 export CODE_PATH=${CODE_PATH:-"/emqx"}
+export SCRIPTS="${CODE_PATH}/scripts"
 export EMQX_NAME=${EMQX_NAME:-"emqx"}
 export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}"
 export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base"
@@ -33,7 +34,7 @@ export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base"
 if [ "$PACKAGE_TYPE" = 'tgz' ]; then
     PKG_SUFFIX="tar.gz"
 else
-    SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)"
+    SYSTEM="$("$SCRIPTS"/get-distro.sh)"
     case "${SYSTEM:-}" in
         ubuntu*|debian*|raspbian*)
             PKG_SUFFIX='deb'
@@ -197,28 +198,30 @@ EOF
 
 relup_test(){
     TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}")"
-    if [ -d "${RELUP_PACKAGE_PATH}" ];then
+    if [ ! -d "${RELUP_PACKAGE_PATH}" ]; then
-        cd "${RELUP_PACKAGE_PATH}"
+        echo "WARNING: ${RELUP_PACKAGE_PATH} is not a dir, skipped relup test!"
-
+        return 0
-        find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.tar.gz" |
+    fi
-            while read -r pkg; do
+    cd "${RELUP_PACKAGE_PATH}"
-                packagename=$(basename "${pkg}")
+    local pattern
-                tar -zxf "$packagename"
+    pattern="$("$SCRIPTS"/pkg-full-vsn.sh "${EMQX_NAME}" 'vsn_matcher')"
-                if ! ./emqx/bin/emqx start; then
+    while read -r pkg; do
-                    cat emqx/log/erlang.log.1 || true
+        packagename=$(basename "${pkg}")
-                    cat emqx/log/emqx.log.1 || true
+        tar -zxf "$packagename"
-                    exit 1
+        if ! ./emqx/bin/emqx start; then
-                fi
+            cat emqx/log/erlang.log.1 || true
-                ./emqx/bin/emqx_ctl status
+            cat emqx/log/emqx.log.1 || true
-                ./emqx/bin/emqx versions
+            exit 1
-                cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".tar.gz ./emqx/releases
+        fi
-                ./emqx/bin/emqx install "${TARGET_VERSION}"
+        ./emqx/bin/emqx_ctl status
-                [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
+        ./emqx/bin/emqx versions
-                ./emqx/bin/emqx_ctl status
+        cp "$PACKAGE_FILE" ./emqx/releases/
-                ./emqx/bin/emqx stop
+        ./emqx/bin/emqx install "${TARGET_VERSION}"
-                rm -rf emqx
+        [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
-            done
+        ./emqx/bin/emqx_ctl status
-   fi
+        ./emqx/bin/emqx stop
+        rm -rf emqx
+    done < <(find . -maxdepth 1 -name "${pattern}.tar.gz")
 }
 
 emqx_prepare

scripts/relup-base-packages.sh

@@ -7,20 +7,17 @@ set -euo pipefail
 
 # ensure dir
 cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+SCRIPTS="$(pwd)/scripts"
 
-PROFILE="${1:-}"
+PROFILE="${1:-emqx}"
-if [ "$PROFILE" = "" ]; then
+export PROFILE
-    PROFILE="emqx"
-fi
-
-OTP_VSN=${OTP_VSN:-$(scripts/get-otp-vsn.sh)}
 
 case $PROFILE in
     "emqx")
         DIR='broker'
         EDITION='community'
         ;;
-    "emqx-ee")
+    "emqx-enterprise")
         DIR='enterprise'
         EDITION='enterprise'
         ;;
@@ -28,33 +25,32 @@ case $PROFILE in
         DIR='edge'
         EDITION='edge'
         ;;
-esac
+    *)
-
+        echo "Unknown profile $PROFILE"
-SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}"
+        exit 1
-
-ARCH="${ARCH:-$(uname -m)}"
-case "$ARCH" in
-    x86_64)
-        ARCH='amd64'
-        ;;
-    aarch64)
-        ARCH='arm64'
-        ;;
-    arm*)
-        ARCH=arm
         ;;
 esac
 
-SHASUM="sha256sum"
+UNAME="$(uname -s)"
-if [ "$SYSTEM" = "macos" ]; then
+case "$UNAME" in
-    SHASUM="shasum -a 256"
+    Darwin)
-fi
+        SHASUM="shasum -a 256"
+        ;;
+    *)
+        SHASUM="sha256sum"
+        ;;
+esac
+
+BASE_VERSIONS="$("${SCRIPTS}"/relup-base-vsns.sh "$EDITION" | xargs echo -n)"
+
+fullvsn() {
+    env PKG_VSN="$1" "${SCRIPTS}"/pkg-full-vsn.sh "$PROFILE"
+}
 
 mkdir -p _upgrade_base
 pushd _upgrade_base >/dev/null
-
+for tag in ${BASE_VERSIONS}; do
-for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do
+    filename="$PROFILE-$(fullvsn "${tag#[e|v]}").tar.gz"
-    filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.tar.gz"
     url="https://www.emqx.com/downloads/$DIR/$tag/$filename"
     echo "downloading ${filename} ..."
     ## if the file does not exist (not downloaded yet)