diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 719549bff..f6f220d2e 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -227,10 +227,9 @@ jobs: - ubuntu16.04 - debian10 - debian9 - # - opensuse + - rockylinux8 - centos7 - raspbian10 - # - raspbian9 build_machine: - aws-arm64 - ubuntu-20.04 @@ -313,11 +312,11 @@ jobs: fi echo "pwd is $PWD" PkgTypes="tgz pkg" - WithElixir="" + IsElixir="no" if [ ${{ matrix.build_elixir }} = "with_elixir" ]; then PkgTypes="tgz" # set Elixir build flag - WithElixir="--with-elixir" + IsElixir="yes" fi for PKGTYPE in ${PkgTypes}; do @@ -326,7 +325,8 @@ jobs: --pkgtype "${PKGTYPE}" \ --arch "${ARCH}" \ --otp "${OTP}" \ - --elixir "${ELIXIR}" "$WithElixir" \ + --elixir "${IsElixir}" \ + --elixir-vsn "${ELIXIR}" \ --system "${SYSTEM}" \ --builder "ghcr.io/emqx/emqx-builder/5.0-7:${ELIXIR}-${OTP}-${SYSTEM}" done diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 22b0f5e84..70948ad65 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -41,7 +41,7 @@ jobs: - 1.13.3 os: - ubuntu20.04 - - centos7 + - rockylinux8 container: "ghcr.io/emqx/emqx-builder/5.0-7:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" @@ -51,8 +51,6 @@ jobs: run: | echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV - echo "EMQX_ELIXIRPKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-elixir${{ matrix.elixir }}-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV - name: Get deps git refs for cache id: deps-refs run: | @@ -75,7 +73,7 @@ jobs: - name: build and test tgz package run: | make ${EMQX_NAME}-tgz - ./scripts/pkg-tests.sh "$EMQX_PKG_NAME" tgz amd64 + ./scripts/pkg-tests.sh ${EMQX_NAME}-tgz - name: run static checks if: contains(matrix.os, 'ubuntu') run: | @@ -83,15 +81,15 @@ jobs: - name: build and test deb/rpm packages run: | make ${EMQX_NAME}-pkg - ./scripts/pkg-tests.sh "$EMQX_PKG_NAME" pkg amd64 + ./scripts/pkg-tests.sh ${EMQX_NAME}-pkg - name: build and test tgz package (Elixir) run: | make ${EMQX_NAME}-elixir-tgz - ./scripts/pkg-tests.sh "$EMQX_ELIXIRPKG_NAME" tgz amd64 + ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-tgz - name: build and test deb/rpm packages (Elixir) run: | - make ${EMQX_NAME}-elixirpkg - ./scripts/pkg-tests.sh "$EMQX_ELIXIRPKG_NAME" pkg amd64 + make ${EMQX_NAME}-elixir-pkg + ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg - uses: actions/upload-artifact@v2 with: name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }} diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index f6da5ad65..733198cd0 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -12,12 +12,18 @@ jobs: strategy: fail-fast: true - + matrix: + profile: + - emqx-enterprise + otp: + - 24.2.1-1 + os: + - ubuntu20.04 steps: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-24.1.5-4-ubuntu20.04 + name: ${{ matrix.profile }}-${{ matrix.otp }}-${{ matrix.os }} workflow: build_slim_packages.yaml path: . - name: Run spellcheck diff --git a/Makefile b/Makefile index 822b06086..666df7ee9 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,6 @@ export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v0.20.0 export DOCKERFILE := deploy/docker/Dockerfile -export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing ifeq ($(OS),Windows_NT) export REBAR_COLOR=none FIND=/usr/bin/find @@ -214,18 +213,6 @@ endef ALL_DOCKERS = $(REL_PROFILES) $(REL_PROFILES:%=%-elixir) $(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt)))) -## emqx-docker-testing -## emqx-enterprise-docker-testing -## is to directly copy a extracted tgz-package to a -## base image such as ubuntu20.04. Mostly for testing -.PHONY: $(REL_PROFILES:%=%-docker-testing) -define gen-docker-target-testing -$1-docker-testing: $(COMMON_DEPS) - @$(BUILD) $1 docker-testing -endef -ALL_TGZS = $(REL_PROFILES) -$(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt)))) - .PHONY: conf-segs: @scripts/merge-config.escript @@ -235,15 +222,15 @@ conf-segs: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get @$(BUILD) $(subst -elixir,,$(@)) elixir -.PHONY: $(REL_PROFILES:%=%-elixirpkg) -define gen-elixirpkg-target +.PHONY: $(REL_PROFILES:%=%-elixir-pkg) +define gen-elixir-pkg-target # the Elixir places the tar in a different path than Rebar3 -$1-elixirpkg: $1-pkg-elixir +$1-elixir-pkg: $1-pkg-elixir @env TAR_PKG_DIR=_build/$1-pkg \ IS_ELIXIR=yes \ $(BUILD) $1-pkg pkg endef -$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixirpkg-target,$(pt)))) +$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixir-pkg-target,$(pt)))) .PHONY: $(REL_PROFILES:%=%-elixir-tgz) define gen-elixir-tgz-target diff --git a/build b/build index 0ab504bc1..675e0d35e 100755 --- a/build +++ b/build @@ -95,13 +95,15 @@ make_relup() { local rel_dir="_build/$PROFILE/rel/emqx" mkdir -p "${rel_dir}/lib" mkdir -p "${rel_dir}/releases" + local name_pattern + name_pattern="${PROFILE}-$(./pkg-vsn.sh "$PROFILE" --vsn_matcher)" local releases=() while read -r tgzfile ; do local base_vsn base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta)\.[0-9])?(-[0-9a-f]{8})?" | head -1)" tar -C "$rel_dir" -zxf ---keep-old-files "$tgzfile" emqx/releases emqx/lib releases+=( "$base_vsn" ) - done < <("$FIND" _upgrade_base -maxdepth 1 -name "$PROFILE*${SYSTEM}-${ARCH}.tar.gz" -type f) + done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.tar.gz" -type f) if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" return 0 @@ -141,15 +143,16 @@ make_tgz() { ELIXIR_MAKE_TAR=yes make_elixir_rel local relpath="_build/${PROFILE}" - target_name="${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" + full_vsn="$(./pkg-vsn.sh "$PROFILE" --long --elixir)" else # build the src_tarball again to ensure relup is included # elixir does not have relup yet. make_rel local relpath="_build/${PROFILE}/rel/emqx" - target_name="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" + full_vsn="$(./pkg-vsn.sh "$PROFILE" --long)" fi + target_name="${PROFILE}-${full_vsn}.tar.gz" target="${pkgpath}/${target_name}" src_tarball="${relpath}/emqx-${PKG_VSN}.tar.gz" @@ -201,46 +204,6 @@ make_docker() { -f "${DOCKERFILE}" . } -## This function accepts any base docker image, -## a emqx tgz-image, and a image tag (for the image to be built), -## to build a docker image which runs EMQX -## -## Export below variables to quickly build an image -## -## Name Default Example -## --------------------------------------------------------------------- -## EMQX_BASE_IMAGE current os centos:7 -## EMQX_TGZ_packages/ /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.tar.gz -## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag -## -make_docker_testing() { - if [ -z "${EMQX_BASE_IMAGE:-}" ]; then - case "$SYSTEM" in - ubuntu20*) - EMQX_BASE_IMAGE="ubuntu:20.04" - ;; - *) - echo "Unsupported testing base image for $SYSTEM" - exit 1 - ;; - esac - fi - EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}" - local default_tgz - default_tgz="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" - local tgz="${EMQX_TGZ_PACKAGE:-$default_tgz}" - if [ ! -f "$tgz" ]; then - log "ERROR: $tgz not built?" - exit 1 - fi - set -x - docker build \ - --build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \ - --build-arg EMQX_TGZ_PACKAGE="${tgz}" \ - --tag "$EMQX_IMAGE_TAG" \ - -f "${DOCKERFILE_TESTING}" . -} - function join { local IFS="$1" shift @@ -310,9 +273,6 @@ case "$ARTIFACT" in docker) make_docker ;; - docker-testing) - make_docker_testing - ;; elixir) make_elixir_rel ;; diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 6ba61c314..a05aec3d0 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -6,16 +6,10 @@ BUILT := $(SRCDIR)/BUILT EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) -ifeq ($(IS_ELIXIR), yes) - ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN) -else - ELIXIR_PKG_VSN := -endif - TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)$(ELIXIR_PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/pkg-vsn.sh $(EMQX_NAME) --long) .PHONY: all all: | $(BUILT) @@ -29,6 +23,9 @@ all: | $(BUILT) cd $(SRCDIR) && dpkg-buildpackage -us -uc mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) cp $(SRCDIR)/../$(SOURCE_PKG).deb $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb + sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb | head -c 64 > \ + $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb.sha256 + $(BUILT): mkdir -p $(TOPDIR) $(SRCDIR) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 4bdf7f6db..6d2dcdf78 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -16,16 +16,10 @@ endif EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) -ifeq ($(IS_ELIXIR), yes) - ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN) -else - ELIXIR_PKG_VSN := -endif - TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) -TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)$(ELIXIR_PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/pkg-vsn.sh $(EMQX_NAME) --long) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C @@ -55,6 +49,8 @@ all: | $(BUILT) emqx.spec mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) cp $(TOPDIR)/RPMS/$(shell uname -m)/$(SOURCE_PKG).rpm $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm + sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm | head -c 64 > \ + $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm.sha256 $(BUILT): mkdir -p $(TOPDIR) $(SRCDIR) $(SRCDIR)/BUILT diff --git a/mix.exs b/mix.exs index 16066f3f5..ece83fd64 100644 --- a/mix.exs +++ b/mix.exs @@ -511,7 +511,7 @@ defmodule EMQXUmbrella.MixProject do # The `:tar` built-in step in Mix Release does not currently add the # `etc` directory into the resulting tarball. The workaround is to # add those to the `:overlays` key before running `:tar`. - # See: https://hexdocs.pm/mix/1.13.2/Mix.Release.html#__struct__/0 + # See: https://hexdocs.pm/mix/1.13.3/Mix.Release.html#__struct__/0 defp prepare_tar_overlays(release) do Map.update!( release, diff --git a/pkg-vsn.sh b/pkg-vsn.sh index 2ba9fc606..ddcf5abe0 100755 --- a/pkg-vsn.sh +++ b/pkg-vsn.sh @@ -6,7 +6,69 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "$0")" -case "${1:-}" in +help() { + echo + echo "$0 PROFILE [options]" + echo + echo "-h|--help: To display this usage information" + echo "--long: Print log vsn number. e.g. 5.0.0-otp24.2.1-1-ubuntu20.04-amd64" + echo " Otherwise short e.g. 5.0.0" + echo "--elixir: Include elixir version in the long version string" + echo " e.g. 5.0.0-elixir1.13.3-otp24.2.1-1-ubuntu20.04-amd64" + echo "--vsn_matcher: For --long option, replace the EMQX version with '*'" + echo " so it can be used in find commands" +} + +PROFILE="${1:-}" +if [ -z "$PROFILE" ]; then + echo "ERROR: missing profile" + help + exit 1 +fi +shift + +while [ "$#" -gt 0 ]; do + case $1 in + -h|--help) + help + exit 0 + ;; + --long) + LONG_VERSION='yes' + shift 1 + ;; + --elixir) + shift 1 + case ${1:-novalue} in + -*) + # another option + IS_ELIXIR='yes' + ;; + yes|no) + IS_ELIXIR="${1}" + shift 1 + ;; + novalue) + IS_ELIXIR='yes' + ;; + *) + echo "ERROR: unknown option: --elixir $2" + exit 1 + ;; + esac + ;; + --vsn_matcher) + IS_MATCHER='yes' + shift 1 + ;; + *) + echo "WARN: Unknown arg (ignored): $1" + exit 1 + ;; + esac +done + +case "${PROFILE}" in *enterprise*) RELEASE_EDITION="EMQX_RELEASE_EE" GIT_TAG_PREFIX="e" @@ -37,4 +99,40 @@ else SUFFIX="-$(git rev-parse HEAD | cut -b1-8)" fi -echo "${RELEASE}${SUFFIX}" +PKG_VSN="${RELEASE}${SUFFIX}" + +if [ "${LONG_VERSION:-}" != 'yes' ]; then + echo "$PKG_VSN" + exit 0 +fi + +### --long LONG_VERSION handling start + +if [ "${IS_MATCHER:-}" = 'yes' ]; then + PKG_VSN='*' +fi + +OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" +SYSTEM="$(./scripts/get-distro.sh)" + +UNAME="$(uname -m)" +case "$UNAME" in + x86_64) + ARCH='amd64' + ;; + aarch64) + ARCH='arm64' + ;; + arm*) + ARCH=arm + ;; +esac + +if [ "${IS_ELIXIR:-}" = "yes" ]; then + ELIXIR_VSN="${ELIXIR_VSN:-$(./scripts/get-elixir-vsn.sh)}" + FULL_VSN="${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +else + FULL_VSN="${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +fi + +echo "${FULL_VSN}" diff --git a/scripts/buildx.sh b/scripts/buildx.sh index afbd2ca63..aebc0b050 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -8,25 +8,26 @@ ## i.e. will not work if docker command has to be executed with sudo ## example: -## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10 +## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \ +## --builder ghcr.io/emqx/emqx-builder/5.0-7:1.13.3-24.2.1-1-debian10 set -euo pipefail help() { echo - echo "-h|--help: To display this usage information" - echo "--profile : EMQX profile to build, e.g. emqx, emqx-edge" - echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz," - echo " pkg for .rpm or .deb" - echo "--with-elixir: Specify if the release should be built with Elixir, " - echo " defaults to false." - echo "--arch amd64|arm64: Target arch to build the EMQX package for" - echo "--src_dir : EMQX source ode in this dir, default to PWD" - echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" - echo "--otp : OTP version being used in the builder" - echo "--elixir : Elixir version being used in the builder" - echo "--system : OS used in the builder image" + echo "-h|--help: To display this usage information" + echo "--profile : EMQX profile to build (emqx|emqx-edge|emqx-enterprise)" + echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz," + echo " pkg for .rpm or .deb" + echo "--elixir: Specify if the release should be built with Elixir, " + echo " defaults to 'no'." + echo "--arch amd64|arm64: Target arch to build the EMQX package for" + echo "--src_dir : EMQX source ode in this dir, default to PWD" + echo "--builder : Builder image to pull" + echo " E.g. ghcr.io/emqx/emqx-builder/5.0-7:1.13.3-24.2.1-1-debian10" + echo "--otp : OTP version being used in the builder" + echo "--elixir-vsn : Elixir version being used in the builder" + echo "--system : OS used in the builder image" } while [ "$#" -gt 0 ]; do @@ -59,13 +60,29 @@ while [ "$#" -gt 0 ]; do OTP_VSN="$2" shift 2 ;; - --elixir) + --elixir-vsn) ELIXIR_VSN="$2" shift 2 ;; - --with-elixir) - WITH_ELIXIR=yes + --elixir) shift 1 + case ${1:-novalue} in + -*) + # another option + IS_ELIXIR='yes' + ;; + yes|no) + IS_ELIXIR="${1}" + shift 1 + ;; + novalue) + IS_ELIXIR='yes' + ;; + *) + echo "ERROR: unknown option: --elixir $2" + exit 1 + ;; + esac ;; --system) SYSTEM="$2" @@ -90,10 +107,13 @@ if [ -z "${PROFILE:-}" ] || exit 1 fi +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + set -x -if [ -z "${WITH_ELIXIR:-}" ]; then - WITH_ELIXIR=no +if [ -z "${IS_ELIXIR:-}" ]; then + IS_ELIXIR=no fi case "$PKGTYPE" in @@ -109,28 +129,15 @@ esac export CODE_PATH="${SRC_DIR:-$PWD}" cd "${CODE_PATH}" -PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" - -if [ "$WITH_ELIXIR" = "yes" ] -then - PKG_NAME="${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +if [ "$IS_ELIXIR" = "yes" ]; then MAKE_TARGET="${PROFILE}-elixir-${PKGTYPE}" else - PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" MAKE_TARGET="${PROFILE}-${PKGTYPE}" fi -CMD_RUN="export EMQX_NAME=\"$PROFILE\"; make ${MAKE_TARGET} && ./scripts/pkg-tests.sh $PKG_NAME $PKGTYPE $ARCH" +CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}" -if docker info; then - docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" - docker run -i --rm \ - -v "$(pwd)":/emqx \ - --workdir /emqx \ - --platform="linux/$ARCH" \ - "$BUILDER" \ - bash -euc "$CMD_RUN" -elif [[ $(uname -m) = "x86_64" && "$ARCH" = "amd64" ]]; then +if [[ $(uname -m) = "x86_64" && "$ARCH" = "amd64" ]]; then eval "$CMD_RUN" elif [[ $(uname -m) = "aarch64" && "$ARCH" = "arm64" ]]; then eval "$CMD_RUN" @@ -138,6 +145,14 @@ elif [[ $(uname -m) = "arm64" && "$ARCH" = "arm64" ]]; then eval "$CMD_RUN" elif [[ $(uname -m) = "armv7l" && "$ARCH" = "arm64" ]]; then eval "$CMD_RUN" +elif docker info; then + docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" + docker run -i --rm \ + -v "$(pwd)":/emqx \ + --workdir /emqx \ + --platform="linux/$ARCH" \ + "$BUILDER" \ + bash -euc "$CMD_RUN" else echo "Error: Docker not available on unsupported platform" exit 1; diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh index 266d49cba..d70151209 100755 --- a/scripts/get-distro.sh +++ b/scripts/get-distro.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash ## This script prints Linux distro name and its version number -## e.g. macos, centos7, ubuntu20.04 +## e.g. macos, el8, ubuntu20.04 set -euo pipefail @@ -14,9 +14,9 @@ case "$UNAME" in SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" ;; Linux) - if grep -q -i 'centos' /etc/*-release; then - DIST='centos' - VERSION_ID="$(rpm --eval '%{centos_ver}')" + if grep -q -i 'rhel' /etc/*-release; then + DIST='el' + VERSION_ID="$(rpm --eval '%{rhel}')" else DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" diff --git a/scripts/pkg-tests.sh b/scripts/pkg-tests.sh index 91c4763d1..6254584a5 100755 --- a/scripts/pkg-tests.sh +++ b/scripts/pkg-tests.sh @@ -3,37 +3,57 @@ set -euo pipefail set -x -if [ -z "${1:-}" ]; then - echo "Usage $0 tgz|pkg" - exit 1 -fi +MAKE_TARGET="${1:-}" -case "${2:-}" in - tgz|pkg) - true - ;; - *) - echo "Usage $0 tgz|pkg" - exit 1 - ;; +case "${MAKE_TARGET}" in + emqx-enterprise-*) + EMQX_NAME='emqx-enterprise' + ;; + emqx-edge-*) + EMQX_NAME='emqx-edge' + ;; + emqx-*) + EMQX_NAME='emqx' + ;; + *) + echo "Usage $0 " + exit 1 + ;; esac -PACKAGE_NAME="${1}" -PACKAGE_TYPE="${2}" -ARCH="${3}" -# Needed by deploy/package/**/Makefile -export ARCH +case "${MAKE_TARGET}" in + *-tgz) + PACKAGE_TYPE='tgz' + ;; + *-pkg) + PACKAGE_TYPE='pkg' + ;; + *) + echo "Unknown package type ${1}" + exit 2 + ;; +esac + +case "${MAKE_TARGET}" in + *elixir*) + IS_ELIXIR='yes' + ;; + *) + IS_ELIXIR='no' + ;; +esac export DEBUG=1 export CODE_PATH=${CODE_PATH:-"/emqx"} -export EMQX_NAME=${EMQX_NAME:-"emqx"} +export SCRIPTS="${CODE_PATH}/scripts" +export EMQX_NAME export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" if [ "$PACKAGE_TYPE" = 'tgz' ]; then PKG_SUFFIX="tar.gz" else - SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)" + SYSTEM="$("$SCRIPTS"/get-distro.sh)" case "${SYSTEM:-}" in ubuntu*|debian*|raspbian*) PKG_SUFFIX='deb' @@ -43,6 +63,9 @@ else ;; esac fi +PACKAGE_VERSION="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}")" +PACKAGE_VERSION_LONG="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}" --long --elixir "${IS_ELIXIR}")" +PACKAGE_NAME="${EMQX_NAME}-${PACKAGE_VERSION_LONG}" PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}" @@ -196,31 +219,36 @@ EOF } relup_test(){ - TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}")" - if [ -d "${RELUP_PACKAGE_PATH}" ];then - cd "${RELUP_PACKAGE_PATH}" - - find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.tar.gz" | - while read -r pkg; do - packagename=$(basename "${pkg}") - tar -zxf "$packagename" - if ! ./emqx/bin/emqx start; then - cat emqx/log/erlang.log.1 || true - cat emqx/log/emqx.log.1 || true - exit 1 - fi - ./emqx/bin/emqx_ctl status - ./emqx/bin/emqx versions - cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".tar.gz ./emqx/releases - ./emqx/bin/emqx install "${TARGET_VERSION}" - [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 - ./emqx/bin/emqx_ctl status - ./emqx/bin/emqx stop - rm -rf emqx - done - fi + if [ ! -d "${RELUP_PACKAGE_PATH}" ]; then + echo "WARNING: ${RELUP_PACKAGE_PATH} is not a dir, skipped relup test!" + return 0 + fi + cd "${RELUP_PACKAGE_PATH}" + local pattern + pattern="$EMQX_NAME-$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}" --long --vsn_matcher)" + while read -r pkg; do + packagename=$(basename "${pkg}") + tar -zxf "$packagename" + if ! ./emqx/bin/emqx start; then + cat emqx/log/erlang.log.1 || true + cat emqx/log/emqx.log.1 || true + exit 1 + fi + ./emqx/bin/emqx_ctl status + ./emqx/bin/emqx versions + cp "$PACKAGE_FILE" ./emqx/releases/ + ./emqx/bin/emqx install "${PACKAGE_VERSION}" + [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${PACKAGE_VERSION}" ] || exit 1 + ./emqx/bin/emqx_ctl status + ./emqx/bin/emqx stop + rm -rf emqx + done < <(find . -maxdepth 1 -name "${pattern}.tar.gz") } emqx_prepare emqx_test -relup_test +if [ "$IS_ELIXIR" = 'yes' ]; then + echo "WARNING: skipped relup test for elixir" +else + relup_test +fi diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 686fcb6c7..ff9e3ba1c 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -7,20 +7,17 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +ROOT_DIR="$(pwd)" -PROFILE="${1:-}" -if [ "$PROFILE" = "" ]; then - PROFILE="emqx" -fi - -OTP_VSN=${OTP_VSN:-$(scripts/get-otp-vsn.sh)} +PROFILE="${1:-emqx}" +export PROFILE case $PROFILE in "emqx") DIR='broker' EDITION='community' ;; - "emqx-ee") + "emqx-enterprise") DIR='enterprise' EDITION='enterprise' ;; @@ -28,33 +25,32 @@ case $PROFILE in DIR='edge' EDITION='edge' ;; -esac - -SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" - -ARCH="${ARCH:-$(uname -m)}" -case "$ARCH" in - x86_64) - ARCH='amd64' - ;; - aarch64) - ARCH='arm64' - ;; - arm*) - ARCH=arm + *) + echo "Unknown profile $PROFILE" + exit 1 ;; esac -SHASUM="sha256sum" -if [ "$SYSTEM" = "macos" ]; then - SHASUM="shasum -a 256" -fi +UNAME="$(uname -s)" +case "$UNAME" in + Darwin) + SHASUM="shasum -a 256" + ;; + *) + SHASUM="sha256sum" + ;; +esac + +BASE_VERSIONS="$("${ROOT_DIR}"/scripts/relup-base-vsns.sh "$EDITION" | xargs echo -n)" + +fullvsn() { + env PKG_VSN="$1" "${ROOT_DIR}"/pkg-vsn.sh "$PROFILE" --long +} mkdir -p _upgrade_base pushd _upgrade_base >/dev/null - -for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do - filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.tar.gz" +for tag in ${BASE_VERSIONS}; do + filename="$PROFILE-$(fullvsn "${tag#[e|v]}").tar.gz" url="https://www.emqx.com/downloads/$DIR/$tag/$filename" echo "downloading ${filename} ..." ## if the file does not exist (not downloaded yet) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index eda28a238..f2b6235e9 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -33,7 +33,7 @@ Options: --release-dir Release directory --src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/' --binary-rel-url Binary release URL pattern. %VSN% variable is substituted with the version in release tag. - E.g. \"https://github.com/emqx/emqx/releases/download/v%VSN%/emqx-%VSN%-otp-24.1.5-3-centos7-amd64.tar.gz\" + E.g. \"https://github.com/emqx/emqx/releases/download/v%VSN%/emqx-%VSN%-otp-24.2.1-1-el7-amd64.tar.gz\" ". -record(app,