From 2d3859a79a06761d3dc4398f0e1a93e8cdcab081 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 24 Feb 2022 13:56:07 +0100 Subject: [PATCH] build: get full version from pkg-full-vsn.sh this is due to the mapping of : centos7 -> el7 and rockylinux8 -> el8 with this fix, there is no need to repeat such mapping in many differennt places --- .github/workflows/spelling.yml | 2 +- Makefile | 13 --------- build | 52 ++++----------------------------- deploy/packages/deb/Makefile | 8 +---- deploy/packages/rpm/Makefile | 8 +---- scripts/buildx.sh | 15 ++++++---- scripts/pkg-full-vsn.sh | 53 ++++++++++++++++++++++++++++++++++ scripts/pkg-tests.sh | 49 ++++++++++++++++--------------- scripts/relup-base-packages.sh | 52 +++++++++++++++------------------ 9 files changed, 122 insertions(+), 130 deletions(-) create mode 100755 scripts/pkg-full-vsn.sh diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index c9c635ba5..733198cd0 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: ${{ matrix.profile }}-${{ mqtrix.otp }}-${{ matrix.os }} + name: ${{ matrix.profile }}-${{ matrix.otp }}-${{ matrix.os }} workflow: build_slim_packages.yaml path: . - name: Run spellcheck diff --git a/Makefile b/Makefile index 822b06086..0f453f0d4 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,6 @@ export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v0.20.0 export DOCKERFILE := deploy/docker/Dockerfile -export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing ifeq ($(OS),Windows_NT) export REBAR_COLOR=none FIND=/usr/bin/find @@ -214,18 +213,6 @@ endef ALL_DOCKERS = $(REL_PROFILES) $(REL_PROFILES:%=%-elixir) $(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt)))) -## emqx-docker-testing -## emqx-enterprise-docker-testing -## is to directly copy a extracted tgz-package to a -## base image such as ubuntu20.04. Mostly for testing -.PHONY: $(REL_PROFILES:%=%-docker-testing) -define gen-docker-target-testing -$1-docker-testing: $(COMMON_DEPS) - @$(BUILD) $1 docker-testing -endef -ALL_TGZS = $(REL_PROFILES) -$(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt)))) - .PHONY: conf-segs: @scripts/merge-config.escript diff --git a/build b/build index 937337484..ce4311365 100755 --- a/build +++ b/build @@ -95,13 +95,15 @@ make_relup() { local rel_dir="_build/$PROFILE/rel/emqx" mkdir -p "${rel_dir}/lib" mkdir -p "${rel_dir}/releases" + local name_pattern + name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh "$PROFILE" 'vsn_matcher')" local releases=() while read -r tgzfile ; do local base_vsn base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta)\.[0-9])?(-[0-9a-f]{8})?" | head -1)" tar -C "$rel_dir" -zxf ---keep-old-files "$tgzfile" emqx/releases emqx/lib releases+=( "$base_vsn" ) - done < <("$FIND" _upgrade_base -maxdepth 1 -name "$PROFILE*${SYSTEM}-${ARCH}.tar.gz" -type f) + done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.tar.gz" -type f) if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" return 0 @@ -141,15 +143,16 @@ make_tgz() { ELIXIR_MAKE_TAR=yes make_elixir_rel local relpath="_build/${PROFILE}" - target_name="${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" + full_vsn="$(WITH_ELIXIR=yes ./scripts/pkg-full-vsn.sh "$PROFILE")" else # build the src_tarball again to ensure relup is included # elixir does not have relup yet. make_rel local relpath="_build/${PROFILE}/rel/emqx" - target_name="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" + full_vsn="$(WITH_ELIXIR=no ./scripts/pkg-full-vsn.sh "$PROFILE")" fi + target_name="${PROFILE}-${full_vsn}.tar.gz" target="${pkgpath}/${target_name}" src_tarball="${relpath}/emqx-${PKG_VSN}.tar.gz" @@ -201,46 +204,6 @@ make_docker() { -f "${DOCKERFILE}" . } -## This function accepts any base docker image, -## a emqx tgz-image, and a image tag (for the image to be built), -## to build a docker image which runs EMQX -## -## Export below variables to quickly build an image -## -## Name Default Example -## --------------------------------------------------------------------- -## EMQX_BASE_IMAGE current os el:7 -## EMQX_TGZ_packages/ /tmp/emqx-4.4.0-otp23.3.4.9-3-el7-amd64.tar.gz -## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag -## -make_docker_testing() { - if [ -z "${EMQX_BASE_IMAGE:-}" ]; then - case "$SYSTEM" in - ubuntu20*) - EMQX_BASE_IMAGE="ubuntu:20.04" - ;; - *) - echo "Unsupported testing base image for $SYSTEM" - exit 1 - ;; - esac - fi - EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}" - local default_tgz - default_tgz="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" - local tgz="${EMQX_TGZ_PACKAGE:-$default_tgz}" - if [ ! -f "$tgz" ]; then - log "ERROR: $tgz not built?" - exit 1 - fi - set -x - docker build \ - --build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \ - --build-arg EMQX_TGZ_PACKAGE="${tgz}" \ - --tag "$EMQX_IMAGE_TAG" \ - -f "${DOCKERFILE_TESTING}" . -} - function join { local IFS="$1" shift @@ -310,9 +273,6 @@ case "$ARTIFACT" in docker) make_docker ;; - docker-testing) - make_docker_testing - ;; elixir) make_elixir_rel ;; diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 6ba61c314..9b8ea0142 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -6,16 +6,10 @@ BUILT := $(SRCDIR)/BUILT EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) -ifeq ($(IS_ELIXIR), yes) - ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN) -else - ELIXIR_PKG_VSN := -endif - TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)$(ELIXIR_PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh $(EMQX_NAME)) .PHONY: all all: | $(BUILT) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 4bdf7f6db..3b7800a6e 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -16,16 +16,10 @@ endif EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) -ifeq ($(IS_ELIXIR), yes) - ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN) -else - ELIXIR_PKG_VSN := -endif - TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) -TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)$(ELIXIR_PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh $(EMQX_NAME)) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C diff --git a/scripts/buildx.sh b/scripts/buildx.sh index a6604ed0a..d72b4bd1f 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -91,6 +91,9 @@ if [ -z "${PROFILE:-}" ] || exit 1 fi +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + set -x if [ -z "${WITH_ELIXIR:-}" ]; then @@ -112,16 +115,18 @@ cd "${CODE_PATH}" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" -if [ "$WITH_ELIXIR" = "yes" ] -then - PKG_NAME="${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +if [ "$WITH_ELIXIR" = "yes" ]; then MAKE_TARGET="${PROFILE}-elixir-${PKGTYPE}" else - PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" MAKE_TARGET="${PROFILE}-${PKGTYPE}" fi -CMD_RUN="export EMQX_NAME=\"$PROFILE\"; make ${MAKE_TARGET} && ./scripts/pkg-tests.sh $PKG_NAME $PKGTYPE $ARCH" +export WITH_ELIXIR +export ELIXIR_VSN +export PROFILE +PKG_NAME="${PROFILE}-$(./scripts/pkg-full-vsn.sh "$PROFILE")" + +CMD_RUN="export EMQX_NAME=\"$PROFILE\"; make ${MAKE_TARGET} && ./scripts/pkg-tests.sh $PKG_NAME $PKGTYPE" if docker info; then docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" diff --git a/scripts/pkg-full-vsn.sh b/scripts/pkg-full-vsn.sh new file mode 100755 index 000000000..146033e38 --- /dev/null +++ b/scripts/pkg-full-vsn.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +## This script print the package full vsn based on current build environment + +## Arg 1 is either 'vsn_exact' (default) or 'vsn_matcher' +## when 'vsn_exact' is given, the version number is the output of pkg-vsn.sh +## otherwise '*' is used for 'find' command to find old versions (as upgrade base) + +set -euo pipefail + +PROFILE="${1:-emqx}" +VSN_MATCH="${2:-vsn_exact}" + +case "${VSN_MATCH}" in + vsn_exact) + PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" + ;; + vsn_matcher) + PKG_VSN='*' + ;; + *) + echo "$0 ERROR: second arg must " + exit 1 + ;; +esac + +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + +OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" +SYSTEM="$(./scripts/get-distro.sh)" + +UNAME="$(uname -m)" +case "$UNAME" in + x86_64) + ARCH='amd64' + ;; + aarch64) + ARCH='arm64' + ;; + arm*) + ARCH=arm + ;; +esac + +if [[ "${WITH_ELIXIR:-}" == "yes" ]] || [[ "${IS_ELIXIR:-}" == "yes" ]] ; then + ELIXIR_VSN="${ELIXIR_VSN:-$(./scripts/get-elixir-vsn.sh)}" + FULL_VSN="${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +else + FULL_VSN="${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +fi + +echo "${FULL_VSN}" diff --git a/scripts/pkg-tests.sh b/scripts/pkg-tests.sh index 91c4763d1..f33d173ea 100755 --- a/scripts/pkg-tests.sh +++ b/scripts/pkg-tests.sh @@ -26,6 +26,7 @@ export ARCH export DEBUG=1 export CODE_PATH=${CODE_PATH:-"/emqx"} +export SCRIPTS="${CODE_PATH}/scripts" export EMQX_NAME=${EMQX_NAME:-"emqx"} export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" @@ -33,7 +34,7 @@ export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" if [ "$PACKAGE_TYPE" = 'tgz' ]; then PKG_SUFFIX="tar.gz" else - SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)" + SYSTEM="$("$SCRIPTS"/get-distro.sh)" case "${SYSTEM:-}" in ubuntu*|debian*|raspbian*) PKG_SUFFIX='deb' @@ -197,28 +198,30 @@ EOF relup_test(){ TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}")" - if [ -d "${RELUP_PACKAGE_PATH}" ];then - cd "${RELUP_PACKAGE_PATH}" - - find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.tar.gz" | - while read -r pkg; do - packagename=$(basename "${pkg}") - tar -zxf "$packagename" - if ! ./emqx/bin/emqx start; then - cat emqx/log/erlang.log.1 || true - cat emqx/log/emqx.log.1 || true - exit 1 - fi - ./emqx/bin/emqx_ctl status - ./emqx/bin/emqx versions - cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".tar.gz ./emqx/releases - ./emqx/bin/emqx install "${TARGET_VERSION}" - [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 - ./emqx/bin/emqx_ctl status - ./emqx/bin/emqx stop - rm -rf emqx - done - fi + if [ ! -d "${RELUP_PACKAGE_PATH}" ]; then + echo "WARNING: ${RELUP_PACKAGE_PATH} is not a dir, skipped relup test!" + return 0 + fi + cd "${RELUP_PACKAGE_PATH}" + local pattern + pattern="$("$SCRIPTS"/pkg-full-vsn.sh "${EMQX_NAME}" 'vsn_matcher')" + while read -r pkg; do + packagename=$(basename "${pkg}") + tar -zxf "$packagename" + if ! ./emqx/bin/emqx start; then + cat emqx/log/erlang.log.1 || true + cat emqx/log/emqx.log.1 || true + exit 1 + fi + ./emqx/bin/emqx_ctl status + ./emqx/bin/emqx versions + cp "$PACKAGE_FILE" ./emqx/releases/ + ./emqx/bin/emqx install "${TARGET_VERSION}" + [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 + ./emqx/bin/emqx_ctl status + ./emqx/bin/emqx stop + rm -rf emqx + done < <(find . -maxdepth 1 -name "${pattern}.tar.gz") } emqx_prepare diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 686fcb6c7..8c9ebfbb1 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -7,20 +7,17 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +SCRIPTS="$(pwd)/scripts" -PROFILE="${1:-}" -if [ "$PROFILE" = "" ]; then - PROFILE="emqx" -fi - -OTP_VSN=${OTP_VSN:-$(scripts/get-otp-vsn.sh)} +PROFILE="${1:-emqx}" +export PROFILE case $PROFILE in "emqx") DIR='broker' EDITION='community' ;; - "emqx-ee") + "emqx-enterprise") DIR='enterprise' EDITION='enterprise' ;; @@ -28,33 +25,32 @@ case $PROFILE in DIR='edge' EDITION='edge' ;; -esac - -SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" - -ARCH="${ARCH:-$(uname -m)}" -case "$ARCH" in - x86_64) - ARCH='amd64' - ;; - aarch64) - ARCH='arm64' - ;; - arm*) - ARCH=arm + *) + echo "Unknown profile $PROFILE" + exit 1 ;; esac -SHASUM="sha256sum" -if [ "$SYSTEM" = "macos" ]; then - SHASUM="shasum -a 256" -fi +UNAME="$(uname -s)" +case "$UNAME" in + Darwin) + SHASUM="shasum -a 256" + ;; + *) + SHASUM="sha256sum" + ;; +esac + +BASE_VERSIONS="$("${SCRIPTS}"/relup-base-vsns.sh "$EDITION" | xargs echo -n)" + +fullvsn() { + env PKG_VSN="$1" "${SCRIPTS}"/pkg-full-vsn.sh "$PROFILE" +} mkdir -p _upgrade_base pushd _upgrade_base >/dev/null - -for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do - filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.tar.gz" +for tag in ${BASE_VERSIONS}; do + filename="$PROFILE-$(fullvsn "${tag#[e|v]}").tar.gz" url="https://www.emqx.com/downloads/$DIR/$tag/$filename" echo "downloading ${filename} ..." ## if the file does not exist (not downloaded yet)