From fa91c7491f2eb81f078dbfde46692d1152f0bf31 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 17 Feb 2022 15:50:17 +0100 Subject: [PATCH 1/7] build: rockylinux --- .github/workflows/build_packages.yaml | 3 +-- .github/workflows/build_slim_packages.yaml | 2 +- build | 4 ++-- scripts/get-distro.sh | 8 ++++---- scripts/update_appup.escript | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 719549bff..7d88af089 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -227,10 +227,9 @@ jobs: - ubuntu16.04 - debian10 - debian9 - # - opensuse + - rockylinux8 - centos7 - raspbian10 - # - raspbian9 build_machine: - aws-arm64 - ubuntu-20.04 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 22b0f5e84..78e0f8f30 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -41,7 +41,7 @@ jobs: - 1.13.3 os: - ubuntu20.04 - - centos7 + - rockylinux8 container: "ghcr.io/emqx/emqx-builder/5.0-7:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" diff --git a/build b/build index 0ab504bc1..937337484 100755 --- a/build +++ b/build @@ -209,8 +209,8 @@ make_docker() { ## ## Name Default Example ## --------------------------------------------------------------------- -## EMQX_BASE_IMAGE current os centos:7 -## EMQX_TGZ_packages/ /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.tar.gz +## EMQX_BASE_IMAGE current os el:7 +## EMQX_TGZ_packages/ /tmp/emqx-4.4.0-otp23.3.4.9-3-el7-amd64.tar.gz ## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag ## make_docker_testing() { diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh index 266d49cba..d70151209 100755 --- a/scripts/get-distro.sh +++ b/scripts/get-distro.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash ## This script prints Linux distro name and its version number -## e.g. macos, centos7, ubuntu20.04 +## e.g. macos, el8, ubuntu20.04 set -euo pipefail @@ -14,9 +14,9 @@ case "$UNAME" in SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" ;; Linux) - if grep -q -i 'centos' /etc/*-release; then - DIST='centos' - VERSION_ID="$(rpm --eval '%{centos_ver}')" + if grep -q -i 'rhel' /etc/*-release; then + DIST='el' + VERSION_ID="$(rpm --eval '%{rhel}')" else DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index eda28a238..e4e55f23c 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -33,7 +33,7 @@ Options: --release-dir Release directory --src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/' --binary-rel-url Binary release URL pattern. %VSN% variable is substituted with the version in release tag. - E.g. \"https://github.com/emqx/emqx/releases/download/v%VSN%/emqx-%VSN%-otp-24.1.5-3-centos7-amd64.tar.gz\" + E.g. \"https://github.com/emqx/emqx/releases/download/v%VSN%/emqx-%VSN%-otp-24.1.5-3-el7-amd64.tar.gz\" ". -record(app, From b2cb7f1bd50f74271a25411f0ca8faff94b9b102 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 23 Feb 2022 18:49:08 +0100 Subject: [PATCH 2/7] build: update to latest builder image --- .github/workflows/spelling.yml | 2 +- mix.exs | 2 +- scripts/buildx.sh | 5 +++-- scripts/update_appup.escript | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index f6da5ad65..c7d4629e5 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-24.1.5-4-ubuntu20.04 + name: emqx-24.2.1-1-ubuntu20.04 workflow: build_slim_packages.yaml path: . - name: Run spellcheck diff --git a/mix.exs b/mix.exs index df0d0f3a3..6c0d52989 100644 --- a/mix.exs +++ b/mix.exs @@ -511,7 +511,7 @@ defmodule EMQXUmbrella.MixProject do # The `:tar` built-in step in Mix Release does not currently add the # `etc` directory into the resulting tarball. The workaround is to # add those to the `:overlays` key before running `:tar`. - # See: https://hexdocs.pm/mix/1.13.2/Mix.Release.html#__struct__/0 + # See: https://hexdocs.pm/mix/1.13.3/Mix.Release.html#__struct__/0 defp prepare_tar_overlays(release) do Map.update!( release, diff --git a/scripts/buildx.sh b/scripts/buildx.sh index afbd2ca63..a6604ed0a 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -8,7 +8,8 @@ ## i.e. will not work if docker command has to be executed with sudo ## example: -## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10 +## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \ +## --builder ghcr.io/emqx/emqx-builder/5.0-7:1.13.3-24.2.1-1-debian10 set -euo pipefail @@ -23,7 +24,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQX package for" echo "--src_dir : EMQX source ode in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/5.0-7:1.13.3=24.2.1-1-debian10" echo "--otp : OTP version being used in the builder" echo "--elixir : Elixir version being used in the builder" echo "--system : OS used in the builder image" diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index e4e55f23c..f2b6235e9 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -33,7 +33,7 @@ Options: --release-dir Release directory --src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/' --binary-rel-url Binary release URL pattern. %VSN% variable is substituted with the version in release tag. - E.g. \"https://github.com/emqx/emqx/releases/download/v%VSN%/emqx-%VSN%-otp-24.1.5-3-el7-amd64.tar.gz\" + E.g. \"https://github.com/emqx/emqx/releases/download/v%VSN%/emqx-%VSN%-otp-24.2.1-1-el7-amd64.tar.gz\" ". -record(app, From 64fd321ad124b0e76ea998405eac0bd5258506fb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 24 Feb 2022 11:56:35 +0100 Subject: [PATCH 3/7] ci(spelling): check enterprise schema --- .github/workflows/spelling.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index c7d4629e5..c9c635ba5 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -12,12 +12,18 @@ jobs: strategy: fail-fast: true - + matrix: + profile: + - emqx-enterprise + otp: + - 24.2.1-1 + os: + - ubuntu20.04 steps: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-24.2.1-1-ubuntu20.04 + name: ${{ matrix.profile }}-${{ mqtrix.otp }}-${{ matrix.os }} workflow: build_slim_packages.yaml path: . - name: Run spellcheck From 2d3859a79a06761d3dc4398f0e1a93e8cdcab081 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 24 Feb 2022 13:56:07 +0100 Subject: [PATCH 4/7] build: get full version from pkg-full-vsn.sh this is due to the mapping of : centos7 -> el7 and rockylinux8 -> el8 with this fix, there is no need to repeat such mapping in many differennt places --- .github/workflows/spelling.yml | 2 +- Makefile | 13 --------- build | 52 ++++----------------------------- deploy/packages/deb/Makefile | 8 +---- deploy/packages/rpm/Makefile | 8 +---- scripts/buildx.sh | 15 ++++++---- scripts/pkg-full-vsn.sh | 53 ++++++++++++++++++++++++++++++++++ scripts/pkg-tests.sh | 49 ++++++++++++++++--------------- scripts/relup-base-packages.sh | 52 +++++++++++++++------------------ 9 files changed, 122 insertions(+), 130 deletions(-) create mode 100755 scripts/pkg-full-vsn.sh diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index c9c635ba5..733198cd0 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: ${{ matrix.profile }}-${{ mqtrix.otp }}-${{ matrix.os }} + name: ${{ matrix.profile }}-${{ matrix.otp }}-${{ matrix.os }} workflow: build_slim_packages.yaml path: . - name: Run spellcheck diff --git a/Makefile b/Makefile index 822b06086..0f453f0d4 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,6 @@ export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v0.20.0 export DOCKERFILE := deploy/docker/Dockerfile -export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing ifeq ($(OS),Windows_NT) export REBAR_COLOR=none FIND=/usr/bin/find @@ -214,18 +213,6 @@ endef ALL_DOCKERS = $(REL_PROFILES) $(REL_PROFILES:%=%-elixir) $(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt)))) -## emqx-docker-testing -## emqx-enterprise-docker-testing -## is to directly copy a extracted tgz-package to a -## base image such as ubuntu20.04. Mostly for testing -.PHONY: $(REL_PROFILES:%=%-docker-testing) -define gen-docker-target-testing -$1-docker-testing: $(COMMON_DEPS) - @$(BUILD) $1 docker-testing -endef -ALL_TGZS = $(REL_PROFILES) -$(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt)))) - .PHONY: conf-segs: @scripts/merge-config.escript diff --git a/build b/build index 937337484..ce4311365 100755 --- a/build +++ b/build @@ -95,13 +95,15 @@ make_relup() { local rel_dir="_build/$PROFILE/rel/emqx" mkdir -p "${rel_dir}/lib" mkdir -p "${rel_dir}/releases" + local name_pattern + name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh "$PROFILE" 'vsn_matcher')" local releases=() while read -r tgzfile ; do local base_vsn base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta)\.[0-9])?(-[0-9a-f]{8})?" | head -1)" tar -C "$rel_dir" -zxf ---keep-old-files "$tgzfile" emqx/releases emqx/lib releases+=( "$base_vsn" ) - done < <("$FIND" _upgrade_base -maxdepth 1 -name "$PROFILE*${SYSTEM}-${ARCH}.tar.gz" -type f) + done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.tar.gz" -type f) if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" return 0 @@ -141,15 +143,16 @@ make_tgz() { ELIXIR_MAKE_TAR=yes make_elixir_rel local relpath="_build/${PROFILE}" - target_name="${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" + full_vsn="$(WITH_ELIXIR=yes ./scripts/pkg-full-vsn.sh "$PROFILE")" else # build the src_tarball again to ensure relup is included # elixir does not have relup yet. make_rel local relpath="_build/${PROFILE}/rel/emqx" - target_name="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" + full_vsn="$(WITH_ELIXIR=no ./scripts/pkg-full-vsn.sh "$PROFILE")" fi + target_name="${PROFILE}-${full_vsn}.tar.gz" target="${pkgpath}/${target_name}" src_tarball="${relpath}/emqx-${PKG_VSN}.tar.gz" @@ -201,46 +204,6 @@ make_docker() { -f "${DOCKERFILE}" . } -## This function accepts any base docker image, -## a emqx tgz-image, and a image tag (for the image to be built), -## to build a docker image which runs EMQX -## -## Export below variables to quickly build an image -## -## Name Default Example -## --------------------------------------------------------------------- -## EMQX_BASE_IMAGE current os el:7 -## EMQX_TGZ_packages/ /tmp/emqx-4.4.0-otp23.3.4.9-3-el7-amd64.tar.gz -## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag -## -make_docker_testing() { - if [ -z "${EMQX_BASE_IMAGE:-}" ]; then - case "$SYSTEM" in - ubuntu20*) - EMQX_BASE_IMAGE="ubuntu:20.04" - ;; - *) - echo "Unsupported testing base image for $SYSTEM" - exit 1 - ;; - esac - fi - EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}" - local default_tgz - default_tgz="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" - local tgz="${EMQX_TGZ_PACKAGE:-$default_tgz}" - if [ ! -f "$tgz" ]; then - log "ERROR: $tgz not built?" - exit 1 - fi - set -x - docker build \ - --build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \ - --build-arg EMQX_TGZ_PACKAGE="${tgz}" \ - --tag "$EMQX_IMAGE_TAG" \ - -f "${DOCKERFILE_TESTING}" . -} - function join { local IFS="$1" shift @@ -310,9 +273,6 @@ case "$ARTIFACT" in docker) make_docker ;; - docker-testing) - make_docker_testing - ;; elixir) make_elixir_rel ;; diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 6ba61c314..9b8ea0142 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -6,16 +6,10 @@ BUILT := $(SRCDIR)/BUILT EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) -ifeq ($(IS_ELIXIR), yes) - ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN) -else - ELIXIR_PKG_VSN := -endif - TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)$(ELIXIR_PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh $(EMQX_NAME)) .PHONY: all all: | $(BUILT) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 4bdf7f6db..3b7800a6e 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -16,16 +16,10 @@ endif EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) -ifeq ($(IS_ELIXIR), yes) - ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN) -else - ELIXIR_PKG_VSN := -endif - TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) -TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)$(ELIXIR_PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh $(EMQX_NAME)) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C diff --git a/scripts/buildx.sh b/scripts/buildx.sh index a6604ed0a..d72b4bd1f 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -91,6 +91,9 @@ if [ -z "${PROFILE:-}" ] || exit 1 fi +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + set -x if [ -z "${WITH_ELIXIR:-}" ]; then @@ -112,16 +115,18 @@ cd "${CODE_PATH}" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" -if [ "$WITH_ELIXIR" = "yes" ] -then - PKG_NAME="${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +if [ "$WITH_ELIXIR" = "yes" ]; then MAKE_TARGET="${PROFILE}-elixir-${PKGTYPE}" else - PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" MAKE_TARGET="${PROFILE}-${PKGTYPE}" fi -CMD_RUN="export EMQX_NAME=\"$PROFILE\"; make ${MAKE_TARGET} && ./scripts/pkg-tests.sh $PKG_NAME $PKGTYPE $ARCH" +export WITH_ELIXIR +export ELIXIR_VSN +export PROFILE +PKG_NAME="${PROFILE}-$(./scripts/pkg-full-vsn.sh "$PROFILE")" + +CMD_RUN="export EMQX_NAME=\"$PROFILE\"; make ${MAKE_TARGET} && ./scripts/pkg-tests.sh $PKG_NAME $PKGTYPE" if docker info; then docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" diff --git a/scripts/pkg-full-vsn.sh b/scripts/pkg-full-vsn.sh new file mode 100755 index 000000000..146033e38 --- /dev/null +++ b/scripts/pkg-full-vsn.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +## This script print the package full vsn based on current build environment + +## Arg 1 is either 'vsn_exact' (default) or 'vsn_matcher' +## when 'vsn_exact' is given, the version number is the output of pkg-vsn.sh +## otherwise '*' is used for 'find' command to find old versions (as upgrade base) + +set -euo pipefail + +PROFILE="${1:-emqx}" +VSN_MATCH="${2:-vsn_exact}" + +case "${VSN_MATCH}" in + vsn_exact) + PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" + ;; + vsn_matcher) + PKG_VSN='*' + ;; + *) + echo "$0 ERROR: second arg must " + exit 1 + ;; +esac + +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + +OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" +SYSTEM="$(./scripts/get-distro.sh)" + +UNAME="$(uname -m)" +case "$UNAME" in + x86_64) + ARCH='amd64' + ;; + aarch64) + ARCH='arm64' + ;; + arm*) + ARCH=arm + ;; +esac + +if [[ "${WITH_ELIXIR:-}" == "yes" ]] || [[ "${IS_ELIXIR:-}" == "yes" ]] ; then + ELIXIR_VSN="${ELIXIR_VSN:-$(./scripts/get-elixir-vsn.sh)}" + FULL_VSN="${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +else + FULL_VSN="${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +fi + +echo "${FULL_VSN}" diff --git a/scripts/pkg-tests.sh b/scripts/pkg-tests.sh index 91c4763d1..f33d173ea 100755 --- a/scripts/pkg-tests.sh +++ b/scripts/pkg-tests.sh @@ -26,6 +26,7 @@ export ARCH export DEBUG=1 export CODE_PATH=${CODE_PATH:-"/emqx"} +export SCRIPTS="${CODE_PATH}/scripts" export EMQX_NAME=${EMQX_NAME:-"emqx"} export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" @@ -33,7 +34,7 @@ export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" if [ "$PACKAGE_TYPE" = 'tgz' ]; then PKG_SUFFIX="tar.gz" else - SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)" + SYSTEM="$("$SCRIPTS"/get-distro.sh)" case "${SYSTEM:-}" in ubuntu*|debian*|raspbian*) PKG_SUFFIX='deb' @@ -197,28 +198,30 @@ EOF relup_test(){ TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}")" - if [ -d "${RELUP_PACKAGE_PATH}" ];then - cd "${RELUP_PACKAGE_PATH}" - - find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.tar.gz" | - while read -r pkg; do - packagename=$(basename "${pkg}") - tar -zxf "$packagename" - if ! ./emqx/bin/emqx start; then - cat emqx/log/erlang.log.1 || true - cat emqx/log/emqx.log.1 || true - exit 1 - fi - ./emqx/bin/emqx_ctl status - ./emqx/bin/emqx versions - cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".tar.gz ./emqx/releases - ./emqx/bin/emqx install "${TARGET_VERSION}" - [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 - ./emqx/bin/emqx_ctl status - ./emqx/bin/emqx stop - rm -rf emqx - done - fi + if [ ! -d "${RELUP_PACKAGE_PATH}" ]; then + echo "WARNING: ${RELUP_PACKAGE_PATH} is not a dir, skipped relup test!" + return 0 + fi + cd "${RELUP_PACKAGE_PATH}" + local pattern + pattern="$("$SCRIPTS"/pkg-full-vsn.sh "${EMQX_NAME}" 'vsn_matcher')" + while read -r pkg; do + packagename=$(basename "${pkg}") + tar -zxf "$packagename" + if ! ./emqx/bin/emqx start; then + cat emqx/log/erlang.log.1 || true + cat emqx/log/emqx.log.1 || true + exit 1 + fi + ./emqx/bin/emqx_ctl status + ./emqx/bin/emqx versions + cp "$PACKAGE_FILE" ./emqx/releases/ + ./emqx/bin/emqx install "${TARGET_VERSION}" + [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 + ./emqx/bin/emqx_ctl status + ./emqx/bin/emqx stop + rm -rf emqx + done < <(find . -maxdepth 1 -name "${pattern}.tar.gz") } emqx_prepare diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 686fcb6c7..8c9ebfbb1 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -7,20 +7,17 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +SCRIPTS="$(pwd)/scripts" -PROFILE="${1:-}" -if [ "$PROFILE" = "" ]; then - PROFILE="emqx" -fi - -OTP_VSN=${OTP_VSN:-$(scripts/get-otp-vsn.sh)} +PROFILE="${1:-emqx}" +export PROFILE case $PROFILE in "emqx") DIR='broker' EDITION='community' ;; - "emqx-ee") + "emqx-enterprise") DIR='enterprise' EDITION='enterprise' ;; @@ -28,33 +25,32 @@ case $PROFILE in DIR='edge' EDITION='edge' ;; -esac - -SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" - -ARCH="${ARCH:-$(uname -m)}" -case "$ARCH" in - x86_64) - ARCH='amd64' - ;; - aarch64) - ARCH='arm64' - ;; - arm*) - ARCH=arm + *) + echo "Unknown profile $PROFILE" + exit 1 ;; esac -SHASUM="sha256sum" -if [ "$SYSTEM" = "macos" ]; then - SHASUM="shasum -a 256" -fi +UNAME="$(uname -s)" +case "$UNAME" in + Darwin) + SHASUM="shasum -a 256" + ;; + *) + SHASUM="sha256sum" + ;; +esac + +BASE_VERSIONS="$("${SCRIPTS}"/relup-base-vsns.sh "$EDITION" | xargs echo -n)" + +fullvsn() { + env PKG_VSN="$1" "${SCRIPTS}"/pkg-full-vsn.sh "$PROFILE" +} mkdir -p _upgrade_base pushd _upgrade_base >/dev/null - -for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do - filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.tar.gz" +for tag in ${BASE_VERSIONS}; do + filename="$PROFILE-$(fullvsn "${tag#[e|v]}").tar.gz" url="https://www.emqx.com/downloads/$DIR/$tag/$filename" echo "downloading ${filename} ..." ## if the file does not exist (not downloaded yet) From 7e575f54c39c73fc691ab388558c8ebbd1c117b3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 24 Feb 2022 17:34:47 +0100 Subject: [PATCH 5/7] build: down prioritise docker buildx --- scripts/buildx.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/buildx.sh b/scripts/buildx.sh index d72b4bd1f..1e475bcf4 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -16,7 +16,7 @@ set -euo pipefail help() { echo echo "-h|--help: To display this usage information" - echo "--profile : EMQX profile to build, e.g. emqx, emqx-edge" + echo "--profile : EMQX profile to build (emqx|emqx-edge|emqx-enterprise)" echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz," echo " pkg for .rpm or .deb" echo "--with-elixir: Specify if the release should be built with Elixir, " @@ -24,7 +24,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQX package for" echo "--src_dir : EMQX source ode in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/5.0-7:1.13.3=24.2.1-1-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/5.0-7:1.13.3-24.2.1-1-debian10" echo "--otp : OTP version being used in the builder" echo "--elixir : Elixir version being used in the builder" echo "--system : OS used in the builder image" @@ -128,15 +128,7 @@ PKG_NAME="${PROFILE}-$(./scripts/pkg-full-vsn.sh "$PROFILE")" CMD_RUN="export EMQX_NAME=\"$PROFILE\"; make ${MAKE_TARGET} && ./scripts/pkg-tests.sh $PKG_NAME $PKGTYPE" -if docker info; then - docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" - docker run -i --rm \ - -v "$(pwd)":/emqx \ - --workdir /emqx \ - --platform="linux/$ARCH" \ - "$BUILDER" \ - bash -euc "$CMD_RUN" -elif [[ $(uname -m) = "x86_64" && "$ARCH" = "amd64" ]]; then +if [[ $(uname -m) = "x86_64" && "$ARCH" = "amd64" ]]; then eval "$CMD_RUN" elif [[ $(uname -m) = "aarch64" && "$ARCH" = "arm64" ]]; then eval "$CMD_RUN" @@ -144,6 +136,14 @@ elif [[ $(uname -m) = "arm64" && "$ARCH" = "arm64" ]]; then eval "$CMD_RUN" elif [[ $(uname -m) = "armv7l" && "$ARCH" = "arm64" ]]; then eval "$CMD_RUN" +elif docker info; then + docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" + docker run -i --rm \ + -v "$(pwd)":/emqx \ + --workdir /emqx \ + --platform="linux/$ARCH" \ + "$BUILDER" \ + bash -euc "$CMD_RUN" else echo "Error: Docker not available on unsupported platform" exit 1; From ea39a9944cb377a235e56d12d95770aaef6cf6e2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 25 Feb 2022 12:19:43 +0100 Subject: [PATCH 6/7] ci: fix package name format for slim build --- .github/workflows/build_slim_packages.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 78e0f8f30..4bdcc474a 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -51,8 +51,8 @@ jobs: run: | echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV - echo "EMQX_ELIXIRPKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-elixir${{ matrix.elixir }}-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV + echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-full-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV + echo "EMQX_ELIXIRPKG_NAME=${{ matrix.profile }}-$(env IS_ELIXIR=yes ./pkg-full-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV - name: Get deps git refs for cache id: deps-refs run: | From b68cb96bd2188bb623f8f965133c94efff34103e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 25 Feb 2022 16:11:05 +0100 Subject: [PATCH 7/7] refactor: make pkg-vsn.sh able to report both short and long version --- .github/workflows/build_packages.yaml | 7 +- .github/workflows/build_slim_packages.yaml | 12 +-- Makefile | 8 +- build | 6 +- deploy/packages/deb/Makefile | 5 +- deploy/packages/rpm/Makefile | 4 +- pkg-vsn.sh | 102 ++++++++++++++++++++- scripts/buildx.sh | 63 +++++++------ scripts/pkg-full-vsn.sh | 53 ----------- scripts/pkg-tests.sh | 71 +++++++++----- scripts/relup-base-packages.sh | 6 +- 11 files changed, 210 insertions(+), 127 deletions(-) delete mode 100755 scripts/pkg-full-vsn.sh diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 7d88af089..f6f220d2e 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -312,11 +312,11 @@ jobs: fi echo "pwd is $PWD" PkgTypes="tgz pkg" - WithElixir="" + IsElixir="no" if [ ${{ matrix.build_elixir }} = "with_elixir" ]; then PkgTypes="tgz" # set Elixir build flag - WithElixir="--with-elixir" + IsElixir="yes" fi for PKGTYPE in ${PkgTypes}; do @@ -325,7 +325,8 @@ jobs: --pkgtype "${PKGTYPE}" \ --arch "${ARCH}" \ --otp "${OTP}" \ - --elixir "${ELIXIR}" "$WithElixir" \ + --elixir "${IsElixir}" \ + --elixir-vsn "${ELIXIR}" \ --system "${SYSTEM}" \ --builder "ghcr.io/emqx/emqx-builder/5.0-7:${ELIXIR}-${OTP}-${SYSTEM}" done diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 4bdcc474a..70948ad65 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -51,8 +51,6 @@ jobs: run: | echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-full-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV - echo "EMQX_ELIXIRPKG_NAME=${{ matrix.profile }}-$(env IS_ELIXIR=yes ./pkg-full-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV - name: Get deps git refs for cache id: deps-refs run: | @@ -75,7 +73,7 @@ jobs: - name: build and test tgz package run: | make ${EMQX_NAME}-tgz - ./scripts/pkg-tests.sh "$EMQX_PKG_NAME" tgz amd64 + ./scripts/pkg-tests.sh ${EMQX_NAME}-tgz - name: run static checks if: contains(matrix.os, 'ubuntu') run: | @@ -83,15 +81,15 @@ jobs: - name: build and test deb/rpm packages run: | make ${EMQX_NAME}-pkg - ./scripts/pkg-tests.sh "$EMQX_PKG_NAME" pkg amd64 + ./scripts/pkg-tests.sh ${EMQX_NAME}-pkg - name: build and test tgz package (Elixir) run: | make ${EMQX_NAME}-elixir-tgz - ./scripts/pkg-tests.sh "$EMQX_ELIXIRPKG_NAME" tgz amd64 + ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-tgz - name: build and test deb/rpm packages (Elixir) run: | - make ${EMQX_NAME}-elixirpkg - ./scripts/pkg-tests.sh "$EMQX_ELIXIRPKG_NAME" pkg amd64 + make ${EMQX_NAME}-elixir-pkg + ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg - uses: actions/upload-artifact@v2 with: name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }} diff --git a/Makefile b/Makefile index 0f453f0d4..666df7ee9 100644 --- a/Makefile +++ b/Makefile @@ -222,15 +222,15 @@ conf-segs: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get @$(BUILD) $(subst -elixir,,$(@)) elixir -.PHONY: $(REL_PROFILES:%=%-elixirpkg) -define gen-elixirpkg-target +.PHONY: $(REL_PROFILES:%=%-elixir-pkg) +define gen-elixir-pkg-target # the Elixir places the tar in a different path than Rebar3 -$1-elixirpkg: $1-pkg-elixir +$1-elixir-pkg: $1-pkg-elixir @env TAR_PKG_DIR=_build/$1-pkg \ IS_ELIXIR=yes \ $(BUILD) $1-pkg pkg endef -$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixirpkg-target,$(pt)))) +$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixir-pkg-target,$(pt)))) .PHONY: $(REL_PROFILES:%=%-elixir-tgz) define gen-elixir-tgz-target diff --git a/build b/build index ce4311365..675e0d35e 100755 --- a/build +++ b/build @@ -96,7 +96,7 @@ make_relup() { mkdir -p "${rel_dir}/lib" mkdir -p "${rel_dir}/releases" local name_pattern - name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh "$PROFILE" 'vsn_matcher')" + name_pattern="${PROFILE}-$(./pkg-vsn.sh "$PROFILE" --vsn_matcher)" local releases=() while read -r tgzfile ; do local base_vsn @@ -143,14 +143,14 @@ make_tgz() { ELIXIR_MAKE_TAR=yes make_elixir_rel local relpath="_build/${PROFILE}" - full_vsn="$(WITH_ELIXIR=yes ./scripts/pkg-full-vsn.sh "$PROFILE")" + full_vsn="$(./pkg-vsn.sh "$PROFILE" --long --elixir)" else # build the src_tarball again to ensure relup is included # elixir does not have relup yet. make_rel local relpath="_build/${PROFILE}/rel/emqx" - full_vsn="$(WITH_ELIXIR=no ./scripts/pkg-full-vsn.sh "$PROFILE")" + full_vsn="$(./pkg-vsn.sh "$PROFILE" --long)" fi target_name="${PROFILE}-${full_vsn}.tar.gz" target="${pkgpath}/${target_name}" diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 9b8ea0142..a05aec3d0 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -9,7 +9,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh $(EMQX_NAME)) +TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/pkg-vsn.sh $(EMQX_NAME) --long) .PHONY: all all: | $(BUILT) @@ -23,6 +23,9 @@ all: | $(BUILT) cd $(SRCDIR) && dpkg-buildpackage -us -uc mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) cp $(SRCDIR)/../$(SOURCE_PKG).deb $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb + sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb | head -c 64 > \ + $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb.sha256 + $(BUILT): mkdir -p $(TOPDIR) $(SRCDIR) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 3b7800a6e..6d2dcdf78 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -19,7 +19,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) -TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh $(EMQX_NAME)) +TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/pkg-vsn.sh $(EMQX_NAME) --long) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C @@ -49,6 +49,8 @@ all: | $(BUILT) emqx.spec mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) cp $(TOPDIR)/RPMS/$(shell uname -m)/$(SOURCE_PKG).rpm $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm + sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm | head -c 64 > \ + $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm.sha256 $(BUILT): mkdir -p $(TOPDIR) $(SRCDIR) $(SRCDIR)/BUILT diff --git a/pkg-vsn.sh b/pkg-vsn.sh index 2ba9fc606..ddcf5abe0 100755 --- a/pkg-vsn.sh +++ b/pkg-vsn.sh @@ -6,7 +6,69 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "$0")" -case "${1:-}" in +help() { + echo + echo "$0 PROFILE [options]" + echo + echo "-h|--help: To display this usage information" + echo "--long: Print log vsn number. e.g. 5.0.0-otp24.2.1-1-ubuntu20.04-amd64" + echo " Otherwise short e.g. 5.0.0" + echo "--elixir: Include elixir version in the long version string" + echo " e.g. 5.0.0-elixir1.13.3-otp24.2.1-1-ubuntu20.04-amd64" + echo "--vsn_matcher: For --long option, replace the EMQX version with '*'" + echo " so it can be used in find commands" +} + +PROFILE="${1:-}" +if [ -z "$PROFILE" ]; then + echo "ERROR: missing profile" + help + exit 1 +fi +shift + +while [ "$#" -gt 0 ]; do + case $1 in + -h|--help) + help + exit 0 + ;; + --long) + LONG_VERSION='yes' + shift 1 + ;; + --elixir) + shift 1 + case ${1:-novalue} in + -*) + # another option + IS_ELIXIR='yes' + ;; + yes|no) + IS_ELIXIR="${1}" + shift 1 + ;; + novalue) + IS_ELIXIR='yes' + ;; + *) + echo "ERROR: unknown option: --elixir $2" + exit 1 + ;; + esac + ;; + --vsn_matcher) + IS_MATCHER='yes' + shift 1 + ;; + *) + echo "WARN: Unknown arg (ignored): $1" + exit 1 + ;; + esac +done + +case "${PROFILE}" in *enterprise*) RELEASE_EDITION="EMQX_RELEASE_EE" GIT_TAG_PREFIX="e" @@ -37,4 +99,40 @@ else SUFFIX="-$(git rev-parse HEAD | cut -b1-8)" fi -echo "${RELEASE}${SUFFIX}" +PKG_VSN="${RELEASE}${SUFFIX}" + +if [ "${LONG_VERSION:-}" != 'yes' ]; then + echo "$PKG_VSN" + exit 0 +fi + +### --long LONG_VERSION handling start + +if [ "${IS_MATCHER:-}" = 'yes' ]; then + PKG_VSN='*' +fi + +OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" +SYSTEM="$(./scripts/get-distro.sh)" + +UNAME="$(uname -m)" +case "$UNAME" in + x86_64) + ARCH='amd64' + ;; + aarch64) + ARCH='arm64' + ;; + arm*) + ARCH=arm + ;; +esac + +if [ "${IS_ELIXIR:-}" = "yes" ]; then + ELIXIR_VSN="${ELIXIR_VSN:-$(./scripts/get-elixir-vsn.sh)}" + FULL_VSN="${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +else + FULL_VSN="${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +fi + +echo "${FULL_VSN}" diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 1e475bcf4..aebc0b050 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -15,19 +15,19 @@ set -euo pipefail help() { echo - echo "-h|--help: To display this usage information" - echo "--profile : EMQX profile to build (emqx|emqx-edge|emqx-enterprise)" - echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz," - echo " pkg for .rpm or .deb" - echo "--with-elixir: Specify if the release should be built with Elixir, " - echo " defaults to false." - echo "--arch amd64|arm64: Target arch to build the EMQX package for" - echo "--src_dir : EMQX source ode in this dir, default to PWD" - echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/5.0-7:1.13.3-24.2.1-1-debian10" - echo "--otp : OTP version being used in the builder" - echo "--elixir : Elixir version being used in the builder" - echo "--system : OS used in the builder image" + echo "-h|--help: To display this usage information" + echo "--profile : EMQX profile to build (emqx|emqx-edge|emqx-enterprise)" + echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz," + echo " pkg for .rpm or .deb" + echo "--elixir: Specify if the release should be built with Elixir, " + echo " defaults to 'no'." + echo "--arch amd64|arm64: Target arch to build the EMQX package for" + echo "--src_dir : EMQX source ode in this dir, default to PWD" + echo "--builder : Builder image to pull" + echo " E.g. ghcr.io/emqx/emqx-builder/5.0-7:1.13.3-24.2.1-1-debian10" + echo "--otp : OTP version being used in the builder" + echo "--elixir-vsn : Elixir version being used in the builder" + echo "--system : OS used in the builder image" } while [ "$#" -gt 0 ]; do @@ -60,13 +60,29 @@ while [ "$#" -gt 0 ]; do OTP_VSN="$2" shift 2 ;; - --elixir) + --elixir-vsn) ELIXIR_VSN="$2" shift 2 ;; - --with-elixir) - WITH_ELIXIR=yes + --elixir) shift 1 + case ${1:-novalue} in + -*) + # another option + IS_ELIXIR='yes' + ;; + yes|no) + IS_ELIXIR="${1}" + shift 1 + ;; + novalue) + IS_ELIXIR='yes' + ;; + *) + echo "ERROR: unknown option: --elixir $2" + exit 1 + ;; + esac ;; --system) SYSTEM="$2" @@ -96,8 +112,8 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." set -x -if [ -z "${WITH_ELIXIR:-}" ]; then - WITH_ELIXIR=no +if [ -z "${IS_ELIXIR:-}" ]; then + IS_ELIXIR=no fi case "$PKGTYPE" in @@ -113,20 +129,13 @@ esac export CODE_PATH="${SRC_DIR:-$PWD}" cd "${CODE_PATH}" -PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" - -if [ "$WITH_ELIXIR" = "yes" ]; then +if [ "$IS_ELIXIR" = "yes" ]; then MAKE_TARGET="${PROFILE}-elixir-${PKGTYPE}" else MAKE_TARGET="${PROFILE}-${PKGTYPE}" fi -export WITH_ELIXIR -export ELIXIR_VSN -export PROFILE -PKG_NAME="${PROFILE}-$(./scripts/pkg-full-vsn.sh "$PROFILE")" - -CMD_RUN="export EMQX_NAME=\"$PROFILE\"; make ${MAKE_TARGET} && ./scripts/pkg-tests.sh $PKG_NAME $PKGTYPE" +CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}" if [[ $(uname -m) = "x86_64" && "$ARCH" = "amd64" ]]; then eval "$CMD_RUN" diff --git a/scripts/pkg-full-vsn.sh b/scripts/pkg-full-vsn.sh deleted file mode 100755 index 146033e38..000000000 --- a/scripts/pkg-full-vsn.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -## This script print the package full vsn based on current build environment - -## Arg 1 is either 'vsn_exact' (default) or 'vsn_matcher' -## when 'vsn_exact' is given, the version number is the output of pkg-vsn.sh -## otherwise '*' is used for 'find' command to find old versions (as upgrade base) - -set -euo pipefail - -PROFILE="${1:-emqx}" -VSN_MATCH="${2:-vsn_exact}" - -case "${VSN_MATCH}" in - vsn_exact) - PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" - ;; - vsn_matcher) - PKG_VSN='*' - ;; - *) - echo "$0 ERROR: second arg must " - exit 1 - ;; -esac - -# ensure dir -cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." - -OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" -SYSTEM="$(./scripts/get-distro.sh)" - -UNAME="$(uname -m)" -case "$UNAME" in - x86_64) - ARCH='amd64' - ;; - aarch64) - ARCH='arm64' - ;; - arm*) - ARCH=arm - ;; -esac - -if [[ "${WITH_ELIXIR:-}" == "yes" ]] || [[ "${IS_ELIXIR:-}" == "yes" ]] ; then - ELIXIR_VSN="${ELIXIR_VSN:-$(./scripts/get-elixir-vsn.sh)}" - FULL_VSN="${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" -else - FULL_VSN="${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" -fi - -echo "${FULL_VSN}" diff --git a/scripts/pkg-tests.sh b/scripts/pkg-tests.sh index f33d173ea..6254584a5 100755 --- a/scripts/pkg-tests.sh +++ b/scripts/pkg-tests.sh @@ -3,31 +3,50 @@ set -euo pipefail set -x -if [ -z "${1:-}" ]; then - echo "Usage $0 tgz|pkg" - exit 1 -fi +MAKE_TARGET="${1:-}" -case "${2:-}" in - tgz|pkg) - true - ;; - *) - echo "Usage $0 tgz|pkg" - exit 1 - ;; +case "${MAKE_TARGET}" in + emqx-enterprise-*) + EMQX_NAME='emqx-enterprise' + ;; + emqx-edge-*) + EMQX_NAME='emqx-edge' + ;; + emqx-*) + EMQX_NAME='emqx' + ;; + *) + echo "Usage $0 " + exit 1 + ;; esac -PACKAGE_NAME="${1}" -PACKAGE_TYPE="${2}" -ARCH="${3}" -# Needed by deploy/package/**/Makefile -export ARCH +case "${MAKE_TARGET}" in + *-tgz) + PACKAGE_TYPE='tgz' + ;; + *-pkg) + PACKAGE_TYPE='pkg' + ;; + *) + echo "Unknown package type ${1}" + exit 2 + ;; +esac + +case "${MAKE_TARGET}" in + *elixir*) + IS_ELIXIR='yes' + ;; + *) + IS_ELIXIR='no' + ;; +esac export DEBUG=1 export CODE_PATH=${CODE_PATH:-"/emqx"} export SCRIPTS="${CODE_PATH}/scripts" -export EMQX_NAME=${EMQX_NAME:-"emqx"} +export EMQX_NAME export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" @@ -44,6 +63,9 @@ else ;; esac fi +PACKAGE_VERSION="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}")" +PACKAGE_VERSION_LONG="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}" --long --elixir "${IS_ELIXIR}")" +PACKAGE_NAME="${EMQX_NAME}-${PACKAGE_VERSION_LONG}" PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}" @@ -197,14 +219,13 @@ EOF } relup_test(){ - TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}")" if [ ! -d "${RELUP_PACKAGE_PATH}" ]; then echo "WARNING: ${RELUP_PACKAGE_PATH} is not a dir, skipped relup test!" return 0 fi cd "${RELUP_PACKAGE_PATH}" local pattern - pattern="$("$SCRIPTS"/pkg-full-vsn.sh "${EMQX_NAME}" 'vsn_matcher')" + pattern="$EMQX_NAME-$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}" --long --vsn_matcher)" while read -r pkg; do packagename=$(basename "${pkg}") tar -zxf "$packagename" @@ -216,8 +237,8 @@ relup_test(){ ./emqx/bin/emqx_ctl status ./emqx/bin/emqx versions cp "$PACKAGE_FILE" ./emqx/releases/ - ./emqx/bin/emqx install "${TARGET_VERSION}" - [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 + ./emqx/bin/emqx install "${PACKAGE_VERSION}" + [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${PACKAGE_VERSION}" ] || exit 1 ./emqx/bin/emqx_ctl status ./emqx/bin/emqx stop rm -rf emqx @@ -226,4 +247,8 @@ relup_test(){ emqx_prepare emqx_test -relup_test +if [ "$IS_ELIXIR" = 'yes' ]; then + echo "WARNING: skipped relup test for elixir" +else + relup_test +fi diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 8c9ebfbb1..ff9e3ba1c 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -7,7 +7,7 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." -SCRIPTS="$(pwd)/scripts" +ROOT_DIR="$(pwd)" PROFILE="${1:-emqx}" export PROFILE @@ -41,10 +41,10 @@ case "$UNAME" in ;; esac -BASE_VERSIONS="$("${SCRIPTS}"/relup-base-vsns.sh "$EDITION" | xargs echo -n)" +BASE_VERSIONS="$("${ROOT_DIR}"/scripts/relup-base-vsns.sh "$EDITION" | xargs echo -n)" fullvsn() { - env PKG_VSN="$1" "${SCRIPTS}"/pkg-full-vsn.sh "$PROFILE" + env PKG_VSN="$1" "${ROOT_DIR}"/pkg-vsn.sh "$PROFILE" --long } mkdir -p _upgrade_base