diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 7d88af089..f6f220d2e 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -312,11 +312,11 @@ jobs: fi echo "pwd is $PWD" PkgTypes="tgz pkg" - WithElixir="" + IsElixir="no" if [ ${{ matrix.build_elixir }} = "with_elixir" ]; then PkgTypes="tgz" # set Elixir build flag - WithElixir="--with-elixir" + IsElixir="yes" fi for PKGTYPE in ${PkgTypes}; do @@ -325,7 +325,8 @@ jobs: --pkgtype "${PKGTYPE}" \ --arch "${ARCH}" \ --otp "${OTP}" \ - --elixir "${ELIXIR}" "$WithElixir" \ + --elixir "${IsElixir}" \ + --elixir-vsn "${ELIXIR}" \ --system "${SYSTEM}" \ --builder "ghcr.io/emqx/emqx-builder/5.0-7:${ELIXIR}-${OTP}-${SYSTEM}" done diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 4bdcc474a..70948ad65 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -51,8 +51,6 @@ jobs: run: | echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-full-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV - echo "EMQX_ELIXIRPKG_NAME=${{ matrix.profile }}-$(env IS_ELIXIR=yes ./pkg-full-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV - name: Get deps git refs for cache id: deps-refs run: | @@ -75,7 +73,7 @@ jobs: - name: build and test tgz package run: | make ${EMQX_NAME}-tgz - ./scripts/pkg-tests.sh "$EMQX_PKG_NAME" tgz amd64 + ./scripts/pkg-tests.sh ${EMQX_NAME}-tgz - name: run static checks if: contains(matrix.os, 'ubuntu') run: | @@ -83,15 +81,15 @@ jobs: - name: build and test deb/rpm packages run: | make ${EMQX_NAME}-pkg - ./scripts/pkg-tests.sh "$EMQX_PKG_NAME" pkg amd64 + ./scripts/pkg-tests.sh ${EMQX_NAME}-pkg - name: build and test tgz package (Elixir) run: | make ${EMQX_NAME}-elixir-tgz - ./scripts/pkg-tests.sh "$EMQX_ELIXIRPKG_NAME" tgz amd64 + ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-tgz - name: build and test deb/rpm packages (Elixir) run: | - make ${EMQX_NAME}-elixirpkg - ./scripts/pkg-tests.sh "$EMQX_ELIXIRPKG_NAME" pkg amd64 + make ${EMQX_NAME}-elixir-pkg + ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg - uses: actions/upload-artifact@v2 with: name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }} diff --git a/Makefile b/Makefile index 0f453f0d4..666df7ee9 100644 --- a/Makefile +++ b/Makefile @@ -222,15 +222,15 @@ conf-segs: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get @$(BUILD) $(subst -elixir,,$(@)) elixir -.PHONY: $(REL_PROFILES:%=%-elixirpkg) -define gen-elixirpkg-target +.PHONY: $(REL_PROFILES:%=%-elixir-pkg) +define gen-elixir-pkg-target # the Elixir places the tar in a different path than Rebar3 -$1-elixirpkg: $1-pkg-elixir +$1-elixir-pkg: $1-pkg-elixir @env TAR_PKG_DIR=_build/$1-pkg \ IS_ELIXIR=yes \ $(BUILD) $1-pkg pkg endef -$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixirpkg-target,$(pt)))) +$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixir-pkg-target,$(pt)))) .PHONY: $(REL_PROFILES:%=%-elixir-tgz) define gen-elixir-tgz-target diff --git a/build b/build index ce4311365..675e0d35e 100755 --- a/build +++ b/build @@ -96,7 +96,7 @@ make_relup() { mkdir -p "${rel_dir}/lib" mkdir -p "${rel_dir}/releases" local name_pattern - name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh "$PROFILE" 'vsn_matcher')" + name_pattern="${PROFILE}-$(./pkg-vsn.sh "$PROFILE" --vsn_matcher)" local releases=() while read -r tgzfile ; do local base_vsn @@ -143,14 +143,14 @@ make_tgz() { ELIXIR_MAKE_TAR=yes make_elixir_rel local relpath="_build/${PROFILE}" - full_vsn="$(WITH_ELIXIR=yes ./scripts/pkg-full-vsn.sh "$PROFILE")" + full_vsn="$(./pkg-vsn.sh "$PROFILE" --long --elixir)" else # build the src_tarball again to ensure relup is included # elixir does not have relup yet. make_rel local relpath="_build/${PROFILE}/rel/emqx" - full_vsn="$(WITH_ELIXIR=no ./scripts/pkg-full-vsn.sh "$PROFILE")" + full_vsn="$(./pkg-vsn.sh "$PROFILE" --long)" fi target_name="${PROFILE}-${full_vsn}.tar.gz" target="${pkgpath}/${target_name}" diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 9b8ea0142..a05aec3d0 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -9,7 +9,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh $(EMQX_NAME)) +TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/pkg-vsn.sh $(EMQX_NAME) --long) .PHONY: all all: | $(BUILT) @@ -23,6 +23,9 @@ all: | $(BUILT) cd $(SRCDIR) && dpkg-buildpackage -us -uc mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) cp $(SRCDIR)/../$(SOURCE_PKG).deb $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb + sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb | head -c 64 > \ + $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb.sha256 + $(BUILT): mkdir -p $(TOPDIR) $(SRCDIR) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 3b7800a6e..6d2dcdf78 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -19,7 +19,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) -TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh $(EMQX_NAME)) +TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/pkg-vsn.sh $(EMQX_NAME) --long) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C @@ -49,6 +49,8 @@ all: | $(BUILT) emqx.spec mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) cp $(TOPDIR)/RPMS/$(shell uname -m)/$(SOURCE_PKG).rpm $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm + sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm | head -c 64 > \ + $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm.sha256 $(BUILT): mkdir -p $(TOPDIR) $(SRCDIR) $(SRCDIR)/BUILT diff --git a/pkg-vsn.sh b/pkg-vsn.sh index 2ba9fc606..ddcf5abe0 100755 --- a/pkg-vsn.sh +++ b/pkg-vsn.sh @@ -6,7 +6,69 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "$0")" -case "${1:-}" in +help() { + echo + echo "$0 PROFILE [options]" + echo + echo "-h|--help: To display this usage information" + echo "--long: Print log vsn number. e.g. 5.0.0-otp24.2.1-1-ubuntu20.04-amd64" + echo " Otherwise short e.g. 5.0.0" + echo "--elixir: Include elixir version in the long version string" + echo " e.g. 5.0.0-elixir1.13.3-otp24.2.1-1-ubuntu20.04-amd64" + echo "--vsn_matcher: For --long option, replace the EMQX version with '*'" + echo " so it can be used in find commands" +} + +PROFILE="${1:-}" +if [ -z "$PROFILE" ]; then + echo "ERROR: missing profile" + help + exit 1 +fi +shift + +while [ "$#" -gt 0 ]; do + case $1 in + -h|--help) + help + exit 0 + ;; + --long) + LONG_VERSION='yes' + shift 1 + ;; + --elixir) + shift 1 + case ${1:-novalue} in + -*) + # another option + IS_ELIXIR='yes' + ;; + yes|no) + IS_ELIXIR="${1}" + shift 1 + ;; + novalue) + IS_ELIXIR='yes' + ;; + *) + echo "ERROR: unknown option: --elixir $2" + exit 1 + ;; + esac + ;; + --vsn_matcher) + IS_MATCHER='yes' + shift 1 + ;; + *) + echo "WARN: Unknown arg (ignored): $1" + exit 1 + ;; + esac +done + +case "${PROFILE}" in *enterprise*) RELEASE_EDITION="EMQX_RELEASE_EE" GIT_TAG_PREFIX="e" @@ -37,4 +99,40 @@ else SUFFIX="-$(git rev-parse HEAD | cut -b1-8)" fi -echo "${RELEASE}${SUFFIX}" +PKG_VSN="${RELEASE}${SUFFIX}" + +if [ "${LONG_VERSION:-}" != 'yes' ]; then + echo "$PKG_VSN" + exit 0 +fi + +### --long LONG_VERSION handling start + +if [ "${IS_MATCHER:-}" = 'yes' ]; then + PKG_VSN='*' +fi + +OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" +SYSTEM="$(./scripts/get-distro.sh)" + +UNAME="$(uname -m)" +case "$UNAME" in + x86_64) + ARCH='amd64' + ;; + aarch64) + ARCH='arm64' + ;; + arm*) + ARCH=arm + ;; +esac + +if [ "${IS_ELIXIR:-}" = "yes" ]; then + ELIXIR_VSN="${ELIXIR_VSN:-$(./scripts/get-elixir-vsn.sh)}" + FULL_VSN="${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +else + FULL_VSN="${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +fi + +echo "${FULL_VSN}" diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 1e475bcf4..aebc0b050 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -15,19 +15,19 @@ set -euo pipefail help() { echo - echo "-h|--help: To display this usage information" - echo "--profile : EMQX profile to build (emqx|emqx-edge|emqx-enterprise)" - echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz," - echo " pkg for .rpm or .deb" - echo "--with-elixir: Specify if the release should be built with Elixir, " - echo " defaults to false." - echo "--arch amd64|arm64: Target arch to build the EMQX package for" - echo "--src_dir : EMQX source ode in this dir, default to PWD" - echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/5.0-7:1.13.3-24.2.1-1-debian10" - echo "--otp : OTP version being used in the builder" - echo "--elixir : Elixir version being used in the builder" - echo "--system : OS used in the builder image" + echo "-h|--help: To display this usage information" + echo "--profile : EMQX profile to build (emqx|emqx-edge|emqx-enterprise)" + echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz," + echo " pkg for .rpm or .deb" + echo "--elixir: Specify if the release should be built with Elixir, " + echo " defaults to 'no'." + echo "--arch amd64|arm64: Target arch to build the EMQX package for" + echo "--src_dir : EMQX source ode in this dir, default to PWD" + echo "--builder : Builder image to pull" + echo " E.g. ghcr.io/emqx/emqx-builder/5.0-7:1.13.3-24.2.1-1-debian10" + echo "--otp : OTP version being used in the builder" + echo "--elixir-vsn : Elixir version being used in the builder" + echo "--system : OS used in the builder image" } while [ "$#" -gt 0 ]; do @@ -60,13 +60,29 @@ while [ "$#" -gt 0 ]; do OTP_VSN="$2" shift 2 ;; - --elixir) + --elixir-vsn) ELIXIR_VSN="$2" shift 2 ;; - --with-elixir) - WITH_ELIXIR=yes + --elixir) shift 1 + case ${1:-novalue} in + -*) + # another option + IS_ELIXIR='yes' + ;; + yes|no) + IS_ELIXIR="${1}" + shift 1 + ;; + novalue) + IS_ELIXIR='yes' + ;; + *) + echo "ERROR: unknown option: --elixir $2" + exit 1 + ;; + esac ;; --system) SYSTEM="$2" @@ -96,8 +112,8 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." set -x -if [ -z "${WITH_ELIXIR:-}" ]; then - WITH_ELIXIR=no +if [ -z "${IS_ELIXIR:-}" ]; then + IS_ELIXIR=no fi case "$PKGTYPE" in @@ -113,20 +129,13 @@ esac export CODE_PATH="${SRC_DIR:-$PWD}" cd "${CODE_PATH}" -PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" - -if [ "$WITH_ELIXIR" = "yes" ]; then +if [ "$IS_ELIXIR" = "yes" ]; then MAKE_TARGET="${PROFILE}-elixir-${PKGTYPE}" else MAKE_TARGET="${PROFILE}-${PKGTYPE}" fi -export WITH_ELIXIR -export ELIXIR_VSN -export PROFILE -PKG_NAME="${PROFILE}-$(./scripts/pkg-full-vsn.sh "$PROFILE")" - -CMD_RUN="export EMQX_NAME=\"$PROFILE\"; make ${MAKE_TARGET} && ./scripts/pkg-tests.sh $PKG_NAME $PKGTYPE" +CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}" if [[ $(uname -m) = "x86_64" && "$ARCH" = "amd64" ]]; then eval "$CMD_RUN" diff --git a/scripts/pkg-full-vsn.sh b/scripts/pkg-full-vsn.sh deleted file mode 100755 index 146033e38..000000000 --- a/scripts/pkg-full-vsn.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -## This script print the package full vsn based on current build environment - -## Arg 1 is either 'vsn_exact' (default) or 'vsn_matcher' -## when 'vsn_exact' is given, the version number is the output of pkg-vsn.sh -## otherwise '*' is used for 'find' command to find old versions (as upgrade base) - -set -euo pipefail - -PROFILE="${1:-emqx}" -VSN_MATCH="${2:-vsn_exact}" - -case "${VSN_MATCH}" in - vsn_exact) - PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" - ;; - vsn_matcher) - PKG_VSN='*' - ;; - *) - echo "$0 ERROR: second arg must " - exit 1 - ;; -esac - -# ensure dir -cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." - -OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" -SYSTEM="$(./scripts/get-distro.sh)" - -UNAME="$(uname -m)" -case "$UNAME" in - x86_64) - ARCH='amd64' - ;; - aarch64) - ARCH='arm64' - ;; - arm*) - ARCH=arm - ;; -esac - -if [[ "${WITH_ELIXIR:-}" == "yes" ]] || [[ "${IS_ELIXIR:-}" == "yes" ]] ; then - ELIXIR_VSN="${ELIXIR_VSN:-$(./scripts/get-elixir-vsn.sh)}" - FULL_VSN="${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" -else - FULL_VSN="${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" -fi - -echo "${FULL_VSN}" diff --git a/scripts/pkg-tests.sh b/scripts/pkg-tests.sh index f33d173ea..6254584a5 100755 --- a/scripts/pkg-tests.sh +++ b/scripts/pkg-tests.sh @@ -3,31 +3,50 @@ set -euo pipefail set -x -if [ -z "${1:-}" ]; then - echo "Usage $0 tgz|pkg" - exit 1 -fi +MAKE_TARGET="${1:-}" -case "${2:-}" in - tgz|pkg) - true - ;; - *) - echo "Usage $0 tgz|pkg" - exit 1 - ;; +case "${MAKE_TARGET}" in + emqx-enterprise-*) + EMQX_NAME='emqx-enterprise' + ;; + emqx-edge-*) + EMQX_NAME='emqx-edge' + ;; + emqx-*) + EMQX_NAME='emqx' + ;; + *) + echo "Usage $0 " + exit 1 + ;; esac -PACKAGE_NAME="${1}" -PACKAGE_TYPE="${2}" -ARCH="${3}" -# Needed by deploy/package/**/Makefile -export ARCH +case "${MAKE_TARGET}" in + *-tgz) + PACKAGE_TYPE='tgz' + ;; + *-pkg) + PACKAGE_TYPE='pkg' + ;; + *) + echo "Unknown package type ${1}" + exit 2 + ;; +esac + +case "${MAKE_TARGET}" in + *elixir*) + IS_ELIXIR='yes' + ;; + *) + IS_ELIXIR='no' + ;; +esac export DEBUG=1 export CODE_PATH=${CODE_PATH:-"/emqx"} export SCRIPTS="${CODE_PATH}/scripts" -export EMQX_NAME=${EMQX_NAME:-"emqx"} +export EMQX_NAME export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" @@ -44,6 +63,9 @@ else ;; esac fi +PACKAGE_VERSION="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}")" +PACKAGE_VERSION_LONG="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}" --long --elixir "${IS_ELIXIR}")" +PACKAGE_NAME="${EMQX_NAME}-${PACKAGE_VERSION_LONG}" PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}" @@ -197,14 +219,13 @@ EOF } relup_test(){ - TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}")" if [ ! -d "${RELUP_PACKAGE_PATH}" ]; then echo "WARNING: ${RELUP_PACKAGE_PATH} is not a dir, skipped relup test!" return 0 fi cd "${RELUP_PACKAGE_PATH}" local pattern - pattern="$("$SCRIPTS"/pkg-full-vsn.sh "${EMQX_NAME}" 'vsn_matcher')" + pattern="$EMQX_NAME-$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}" --long --vsn_matcher)" while read -r pkg; do packagename=$(basename "${pkg}") tar -zxf "$packagename" @@ -216,8 +237,8 @@ relup_test(){ ./emqx/bin/emqx_ctl status ./emqx/bin/emqx versions cp "$PACKAGE_FILE" ./emqx/releases/ - ./emqx/bin/emqx install "${TARGET_VERSION}" - [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 + ./emqx/bin/emqx install "${PACKAGE_VERSION}" + [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${PACKAGE_VERSION}" ] || exit 1 ./emqx/bin/emqx_ctl status ./emqx/bin/emqx stop rm -rf emqx @@ -226,4 +247,8 @@ relup_test(){ emqx_prepare emqx_test -relup_test +if [ "$IS_ELIXIR" = 'yes' ]; then + echo "WARNING: skipped relup test for elixir" +else + relup_test +fi diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 8c9ebfbb1..ff9e3ba1c 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -7,7 +7,7 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." -SCRIPTS="$(pwd)/scripts" +ROOT_DIR="$(pwd)" PROFILE="${1:-emqx}" export PROFILE @@ -41,10 +41,10 @@ case "$UNAME" in ;; esac -BASE_VERSIONS="$("${SCRIPTS}"/relup-base-vsns.sh "$EDITION" | xargs echo -n)" +BASE_VERSIONS="$("${ROOT_DIR}"/scripts/relup-base-vsns.sh "$EDITION" | xargs echo -n)" fullvsn() { - env PKG_VSN="$1" "${SCRIPTS}"/pkg-full-vsn.sh "$PROFILE" + env PKG_VSN="$1" "${ROOT_DIR}"/pkg-vsn.sh "$PROFILE" --long } mkdir -p _upgrade_base