refactor: make pkg-vsn.sh able to report both short and long version

This commit is contained in:
Zaiming (Stone) Shi 2022-02-25 16:11:05 +01:00
parent ea39a9944c
commit b68cb96bd2
11 changed files with 210 additions and 127 deletions

View File

@ -312,11 +312,11 @@ jobs:
fi
echo "pwd is $PWD"
PkgTypes="tgz pkg"
WithElixir=""
IsElixir="no"
if [ ${{ matrix.build_elixir }} = "with_elixir" ]; then
PkgTypes="tgz"
# set Elixir build flag
WithElixir="--with-elixir"
IsElixir="yes"
fi
for PKGTYPE in ${PkgTypes};
do
@ -325,7 +325,8 @@ jobs:
--pkgtype "${PKGTYPE}" \
--arch "${ARCH}" \
--otp "${OTP}" \
--elixir "${ELIXIR}" "$WithElixir" \
--elixir "${IsElixir}" \
--elixir-vsn "${ELIXIR}" \
--system "${SYSTEM}" \
--builder "ghcr.io/emqx/emqx-builder/5.0-7:${ELIXIR}-${OTP}-${SYSTEM}"
done

View File

@ -51,8 +51,6 @@ jobs:
run: |
echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-full-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV
echo "EMQX_ELIXIRPKG_NAME=${{ matrix.profile }}-$(env IS_ELIXIR=yes ./pkg-full-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV
- name: Get deps git refs for cache
id: deps-refs
run: |
@ -75,7 +73,7 @@ jobs:
- name: build and test tgz package
run: |
make ${EMQX_NAME}-tgz
./scripts/pkg-tests.sh "$EMQX_PKG_NAME" tgz amd64
./scripts/pkg-tests.sh ${EMQX_NAME}-tgz
- name: run static checks
if: contains(matrix.os, 'ubuntu')
run: |
@ -83,15 +81,15 @@ jobs:
- name: build and test deb/rpm packages
run: |
make ${EMQX_NAME}-pkg
./scripts/pkg-tests.sh "$EMQX_PKG_NAME" pkg amd64
./scripts/pkg-tests.sh ${EMQX_NAME}-pkg
- name: build and test tgz package (Elixir)
run: |
make ${EMQX_NAME}-elixir-tgz
./scripts/pkg-tests.sh "$EMQX_ELIXIRPKG_NAME" tgz amd64
./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-tgz
- name: build and test deb/rpm packages (Elixir)
run: |
make ${EMQX_NAME}-elixirpkg
./scripts/pkg-tests.sh "$EMQX_ELIXIRPKG_NAME" pkg amd64
make ${EMQX_NAME}-elixir-pkg
./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg
- uses: actions/upload-artifact@v2
with:
name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }}

View File

@ -222,15 +222,15 @@ conf-segs:
$(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get
@$(BUILD) $(subst -elixir,,$(@)) elixir
.PHONY: $(REL_PROFILES:%=%-elixirpkg)
define gen-elixirpkg-target
.PHONY: $(REL_PROFILES:%=%-elixir-pkg)
define gen-elixir-pkg-target
# the Elixir places the tar in a different path than Rebar3
$1-elixirpkg: $1-pkg-elixir
$1-elixir-pkg: $1-pkg-elixir
@env TAR_PKG_DIR=_build/$1-pkg \
IS_ELIXIR=yes \
$(BUILD) $1-pkg pkg
endef
$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixirpkg-target,$(pt))))
$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixir-pkg-target,$(pt))))
.PHONY: $(REL_PROFILES:%=%-elixir-tgz)
define gen-elixir-tgz-target

6
build
View File

@ -96,7 +96,7 @@ make_relup() {
mkdir -p "${rel_dir}/lib"
mkdir -p "${rel_dir}/releases"
local name_pattern
name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh "$PROFILE" 'vsn_matcher')"
name_pattern="${PROFILE}-$(./pkg-vsn.sh "$PROFILE" --vsn_matcher)"
local releases=()
while read -r tgzfile ; do
local base_vsn
@ -143,14 +143,14 @@ make_tgz() {
ELIXIR_MAKE_TAR=yes make_elixir_rel
local relpath="_build/${PROFILE}"
full_vsn="$(WITH_ELIXIR=yes ./scripts/pkg-full-vsn.sh "$PROFILE")"
full_vsn="$(./pkg-vsn.sh "$PROFILE" --long --elixir)"
else
# build the src_tarball again to ensure relup is included
# elixir does not have relup yet.
make_rel
local relpath="_build/${PROFILE}/rel/emqx"
full_vsn="$(WITH_ELIXIR=no ./scripts/pkg-full-vsn.sh "$PROFILE")"
full_vsn="$(./pkg-vsn.sh "$PROFILE" --long)"
fi
target_name="${PROFILE}-${full_vsn}.tar.gz"
target="${pkgpath}/${target_name}"

View File

@ -9,7 +9,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx
TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz
SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture)
TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh $(EMQX_NAME))
TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/pkg-vsn.sh $(EMQX_NAME) --long)
.PHONY: all
all: | $(BUILT)
@ -23,6 +23,9 @@ all: | $(BUILT)
cd $(SRCDIR) && dpkg-buildpackage -us -uc
mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME)
cp $(SRCDIR)/../$(SOURCE_PKG).deb $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb
sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb | head -c 64 > \
$(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb.sha256
$(BUILT):
mkdir -p $(TOPDIR) $(SRCDIR)

View File

@ -19,7 +19,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx
TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz
SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh $(EMQX_NAME))
TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/pkg-vsn.sh $(EMQX_NAME) --long)
SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi)
# Not $(PWD) as it does not work for make -C
@ -49,6 +49,8 @@ all: | $(BUILT)
emqx.spec
mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME)
cp $(TOPDIR)/RPMS/$(shell uname -m)/$(SOURCE_PKG).rpm $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm
sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm | head -c 64 > \
$(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm.sha256
$(BUILT):
mkdir -p $(TOPDIR) $(SRCDIR) $(SRCDIR)/BUILT

View File

@ -6,7 +6,69 @@ set -euo pipefail
# ensure dir
cd -P -- "$(dirname -- "$0")"
case "${1:-}" in
help() {
echo
echo "$0 PROFILE [options]"
echo
echo "-h|--help: To display this usage information"
echo "--long: Print log vsn number. e.g. 5.0.0-otp24.2.1-1-ubuntu20.04-amd64"
echo " Otherwise short e.g. 5.0.0"
echo "--elixir: Include elixir version in the long version string"
echo " e.g. 5.0.0-elixir1.13.3-otp24.2.1-1-ubuntu20.04-amd64"
echo "--vsn_matcher: For --long option, replace the EMQX version with '*'"
echo " so it can be used in find commands"
}
PROFILE="${1:-}"
if [ -z "$PROFILE" ]; then
echo "ERROR: missing profile"
help
exit 1
fi
shift
while [ "$#" -gt 0 ]; do
case $1 in
-h|--help)
help
exit 0
;;
--long)
LONG_VERSION='yes'
shift 1
;;
--elixir)
shift 1
case ${1:-novalue} in
-*)
# another option
IS_ELIXIR='yes'
;;
yes|no)
IS_ELIXIR="${1}"
shift 1
;;
novalue)
IS_ELIXIR='yes'
;;
*)
echo "ERROR: unknown option: --elixir $2"
exit 1
;;
esac
;;
--vsn_matcher)
IS_MATCHER='yes'
shift 1
;;
*)
echo "WARN: Unknown arg (ignored): $1"
exit 1
;;
esac
done
case "${PROFILE}" in
*enterprise*)
RELEASE_EDITION="EMQX_RELEASE_EE"
GIT_TAG_PREFIX="e"
@ -37,4 +99,40 @@ else
SUFFIX="-$(git rev-parse HEAD | cut -b1-8)"
fi
echo "${RELEASE}${SUFFIX}"
PKG_VSN="${RELEASE}${SUFFIX}"
if [ "${LONG_VERSION:-}" != 'yes' ]; then
echo "$PKG_VSN"
exit 0
fi
### --long LONG_VERSION handling start
if [ "${IS_MATCHER:-}" = 'yes' ]; then
PKG_VSN='*'
fi
OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}"
SYSTEM="$(./scripts/get-distro.sh)"
UNAME="$(uname -m)"
case "$UNAME" in
x86_64)
ARCH='amd64'
;;
aarch64)
ARCH='arm64'
;;
arm*)
ARCH=arm
;;
esac
if [ "${IS_ELIXIR:-}" = "yes" ]; then
ELIXIR_VSN="${ELIXIR_VSN:-$(./scripts/get-elixir-vsn.sh)}"
FULL_VSN="${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
else
FULL_VSN="${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
fi
echo "${FULL_VSN}"

View File

@ -15,19 +15,19 @@ set -euo pipefail
help() {
echo
echo "-h|--help: To display this usage information"
echo "--profile <PROFILE>: EMQX profile to build (emqx|emqx-edge|emqx-enterprise)"
echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz,"
echo " pkg for .rpm or .deb"
echo "--with-elixir: Specify if the release should be built with Elixir, "
echo " defaults to false."
echo "--arch amd64|arm64: Target arch to build the EMQX package for"
echo "--src_dir <SRC_DIR>: EMQX source ode in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull"
echo " E.g. ghcr.io/emqx/emqx-builder/5.0-7:1.13.3-24.2.1-1-debian10"
echo "--otp <OTP_VSN>: OTP version being used in the builder"
echo "--elixir <ELIXIR_VSN>: Elixir version being used in the builder"
echo "--system <SYSTEM>: OS used in the builder image"
echo "-h|--help: To display this usage information"
echo "--profile <PROFILE>: EMQX profile to build (emqx|emqx-edge|emqx-enterprise)"
echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz,"
echo " pkg for .rpm or .deb"
echo "--elixir: Specify if the release should be built with Elixir, "
echo " defaults to 'no'."
echo "--arch amd64|arm64: Target arch to build the EMQX package for"
echo "--src_dir <SRC_DIR>: EMQX source ode in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull"
echo " E.g. ghcr.io/emqx/emqx-builder/5.0-7:1.13.3-24.2.1-1-debian10"
echo "--otp <OTP_VSN>: OTP version being used in the builder"
echo "--elixir-vsn <ELIXIR_VSN>: Elixir version being used in the builder"
echo "--system <SYSTEM>: OS used in the builder image"
}
while [ "$#" -gt 0 ]; do
@ -60,13 +60,29 @@ while [ "$#" -gt 0 ]; do
OTP_VSN="$2"
shift 2
;;
--elixir)
--elixir-vsn)
ELIXIR_VSN="$2"
shift 2
;;
--with-elixir)
WITH_ELIXIR=yes
--elixir)
shift 1
case ${1:-novalue} in
-*)
# another option
IS_ELIXIR='yes'
;;
yes|no)
IS_ELIXIR="${1}"
shift 1
;;
novalue)
IS_ELIXIR='yes'
;;
*)
echo "ERROR: unknown option: --elixir $2"
exit 1
;;
esac
;;
--system)
SYSTEM="$2"
@ -96,8 +112,8 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
set -x
if [ -z "${WITH_ELIXIR:-}" ]; then
WITH_ELIXIR=no
if [ -z "${IS_ELIXIR:-}" ]; then
IS_ELIXIR=no
fi
case "$PKGTYPE" in
@ -113,20 +129,13 @@ esac
export CODE_PATH="${SRC_DIR:-$PWD}"
cd "${CODE_PATH}"
PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}"
if [ "$WITH_ELIXIR" = "yes" ]; then
if [ "$IS_ELIXIR" = "yes" ]; then
MAKE_TARGET="${PROFILE}-elixir-${PKGTYPE}"
else
MAKE_TARGET="${PROFILE}-${PKGTYPE}"
fi
export WITH_ELIXIR
export ELIXIR_VSN
export PROFILE
PKG_NAME="${PROFILE}-$(./scripts/pkg-full-vsn.sh "$PROFILE")"
CMD_RUN="export EMQX_NAME=\"$PROFILE\"; make ${MAKE_TARGET} && ./scripts/pkg-tests.sh $PKG_NAME $PKGTYPE"
CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}"
if [[ $(uname -m) = "x86_64" && "$ARCH" = "amd64" ]]; then
eval "$CMD_RUN"

View File

@ -1,53 +0,0 @@
#!/usr/bin/env bash
## This script print the package full vsn based on current build environment
## Arg 1 is either 'vsn_exact' (default) or 'vsn_matcher'
## when 'vsn_exact' is given, the version number is the output of pkg-vsn.sh
## otherwise '*' is used for 'find' command to find old versions (as upgrade base)
set -euo pipefail
PROFILE="${1:-emqx}"
VSN_MATCH="${2:-vsn_exact}"
case "${VSN_MATCH}" in
vsn_exact)
PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}"
;;
vsn_matcher)
PKG_VSN='*'
;;
*)
echo "$0 ERROR: second arg must "
exit 1
;;
esac
# ensure dir
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}"
SYSTEM="$(./scripts/get-distro.sh)"
UNAME="$(uname -m)"
case "$UNAME" in
x86_64)
ARCH='amd64'
;;
aarch64)
ARCH='arm64'
;;
arm*)
ARCH=arm
;;
esac
if [[ "${WITH_ELIXIR:-}" == "yes" ]] || [[ "${IS_ELIXIR:-}" == "yes" ]] ; then
ELIXIR_VSN="${ELIXIR_VSN:-$(./scripts/get-elixir-vsn.sh)}"
FULL_VSN="${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
else
FULL_VSN="${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
fi
echo "${FULL_VSN}"

View File

@ -3,31 +3,50 @@
set -euo pipefail
set -x
if [ -z "${1:-}" ]; then
echo "Usage $0 <PACKAGE_NAME> tgz|pkg"
exit 1
fi
MAKE_TARGET="${1:-}"
case "${2:-}" in
tgz|pkg)
true
;;
*)
echo "Usage $0 <PACKAGE_NAME> tgz|pkg"
exit 1
;;
case "${MAKE_TARGET}" in
emqx-enterprise-*)
EMQX_NAME='emqx-enterprise'
;;
emqx-edge-*)
EMQX_NAME='emqx-edge'
;;
emqx-*)
EMQX_NAME='emqx'
;;
*)
echo "Usage $0 <PKG_TARGET>"
exit 1
;;
esac
PACKAGE_NAME="${1}"
PACKAGE_TYPE="${2}"
ARCH="${3}"
# Needed by deploy/package/**/Makefile
export ARCH
case "${MAKE_TARGET}" in
*-tgz)
PACKAGE_TYPE='tgz'
;;
*-pkg)
PACKAGE_TYPE='pkg'
;;
*)
echo "Unknown package type ${1}"
exit 2
;;
esac
case "${MAKE_TARGET}" in
*elixir*)
IS_ELIXIR='yes'
;;
*)
IS_ELIXIR='no'
;;
esac
export DEBUG=1
export CODE_PATH=${CODE_PATH:-"/emqx"}
export SCRIPTS="${CODE_PATH}/scripts"
export EMQX_NAME=${EMQX_NAME:-"emqx"}
export EMQX_NAME
export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}"
export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base"
@ -44,6 +63,9 @@ else
;;
esac
fi
PACKAGE_VERSION="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}")"
PACKAGE_VERSION_LONG="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}" --long --elixir "${IS_ELIXIR}")"
PACKAGE_NAME="${EMQX_NAME}-${PACKAGE_VERSION_LONG}"
PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}"
PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}"
@ -197,14 +219,13 @@ EOF
}
relup_test(){
TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}")"
if [ ! -d "${RELUP_PACKAGE_PATH}" ]; then
echo "WARNING: ${RELUP_PACKAGE_PATH} is not a dir, skipped relup test!"
return 0
fi
cd "${RELUP_PACKAGE_PATH}"
local pattern
pattern="$("$SCRIPTS"/pkg-full-vsn.sh "${EMQX_NAME}" 'vsn_matcher')"
pattern="$EMQX_NAME-$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}" --long --vsn_matcher)"
while read -r pkg; do
packagename=$(basename "${pkg}")
tar -zxf "$packagename"
@ -216,8 +237,8 @@ relup_test(){
./emqx/bin/emqx_ctl status
./emqx/bin/emqx versions
cp "$PACKAGE_FILE" ./emqx/releases/
./emqx/bin/emqx install "${TARGET_VERSION}"
[ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
./emqx/bin/emqx install "${PACKAGE_VERSION}"
[ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${PACKAGE_VERSION}" ] || exit 1
./emqx/bin/emqx_ctl status
./emqx/bin/emqx stop
rm -rf emqx
@ -226,4 +247,8 @@ relup_test(){
emqx_prepare
emqx_test
relup_test
if [ "$IS_ELIXIR" = 'yes' ]; then
echo "WARNING: skipped relup test for elixir"
else
relup_test
fi

View File

@ -7,7 +7,7 @@ set -euo pipefail
# ensure dir
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
SCRIPTS="$(pwd)/scripts"
ROOT_DIR="$(pwd)"
PROFILE="${1:-emqx}"
export PROFILE
@ -41,10 +41,10 @@ case "$UNAME" in
;;
esac
BASE_VERSIONS="$("${SCRIPTS}"/relup-base-vsns.sh "$EDITION" | xargs echo -n)"
BASE_VERSIONS="$("${ROOT_DIR}"/scripts/relup-base-vsns.sh "$EDITION" | xargs echo -n)"
fullvsn() {
env PKG_VSN="$1" "${SCRIPTS}"/pkg-full-vsn.sh "$PROFILE"
env PKG_VSN="$1" "${ROOT_DIR}"/pkg-vsn.sh "$PROFILE" --long
}
mkdir -p _upgrade_base