474 lines
14 KiB
Bash
Executable File
474 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# This script helps to build release artifacts.
|
|
# arg1: profile, e.g. emqx | emqx-enterprise
|
|
# arg2: artifact, e.g. rel | relup | tgz | pkg
|
|
|
|
set -euo pipefail
|
|
|
|
[ "${DEBUG:-0}" -eq 1 ] && set -x
|
|
|
|
PROFILE_ARG="$1"
|
|
ARTIFACT="$2"
|
|
|
|
is_enterprise() {
|
|
case "$1" in
|
|
*enterprise*)
|
|
echo 'yes'
|
|
;;
|
|
*)
|
|
echo 'no'
|
|
;;
|
|
esac
|
|
}
|
|
PROFILE_ENV="${PROFILE:-${PROFILE_ARG}}"
|
|
case "$(is_enterprise "$PROFILE_ARG"),$(is_enterprise "$PROFILE_ENV")" in
|
|
'yes,yes')
|
|
true
|
|
;;
|
|
'no,no')
|
|
true
|
|
;;
|
|
*)
|
|
echo "PROFILE env var is set to '$PROFILE_ENV', but '$0' arg1 is '$PROFILE_ARG'"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
# make sure PROFILE is exported, it is needed by rebar.config.erl
|
|
PROFILE=$PROFILE_ARG
|
|
export PROFILE
|
|
|
|
# ensure dir
|
|
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
|
|
|
|
PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}"
|
|
export PKG_VSN
|
|
|
|
SYSTEM="$(./scripts/get-distro.sh)"
|
|
|
|
ARCH="$(uname -m)"
|
|
case "$ARCH" in
|
|
x86_64)
|
|
ARCH='amd64'
|
|
;;
|
|
aarch64)
|
|
ARCH='arm64'
|
|
;;
|
|
arm*)
|
|
ARCH='arm64'
|
|
;;
|
|
esac
|
|
export ARCH
|
|
|
|
##
|
|
## Support RPM and Debian based linux systems
|
|
##
|
|
if [ "$(uname -s)" = 'Linux' ]; then
|
|
case "${SYSTEM:-}" in
|
|
ubuntu*|debian*|raspbian*)
|
|
PKGERDIR='deb'
|
|
;;
|
|
*)
|
|
PKGERDIR='rpm'
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
if [ "${SYSTEM}" = 'windows' ]; then
|
|
# windows does not like the find
|
|
FIND="/usr/bin/find"
|
|
TAR="/usr/bin/tar"
|
|
export BUILD_WITHOUT_ROCKSDB="on"
|
|
else
|
|
FIND='find'
|
|
TAR='tar'
|
|
fi
|
|
|
|
log() {
|
|
local msg="$1"
|
|
# rebar3 prints ===>, so we print ===<
|
|
echo "===< $msg"
|
|
}
|
|
|
|
prepare_erl_libs() {
|
|
local libs_dir="$1"
|
|
local erl_libs="${ERL_LIBS:-}"
|
|
local sep
|
|
if [ "${SYSTEM}" = 'windows' ]; then
|
|
sep=';'
|
|
else
|
|
sep=':'
|
|
fi
|
|
for app in "${libs_dir}"/*; do
|
|
if [ -d "${app}/ebin" ]; then
|
|
if [ -n "$erl_libs" ]; then
|
|
erl_libs="${erl_libs}${sep}${app}"
|
|
else
|
|
erl_libs="${app}"
|
|
fi
|
|
fi
|
|
done
|
|
export ERL_LIBS="$erl_libs"
|
|
}
|
|
|
|
make_docs() {
|
|
case "$(is_enterprise "$PROFILE")" in
|
|
'yes')
|
|
SCHEMA_MODULE='emqx_enterprise_schema'
|
|
;;
|
|
'no')
|
|
SCHEMA_MODULE='emqx_conf_schema'
|
|
;;
|
|
esac
|
|
prepare_erl_libs "_build/$PROFILE/checkouts"
|
|
prepare_erl_libs "_build/$PROFILE/lib"
|
|
local docdir="_build/docgen/$PROFILE"
|
|
mkdir -p "$docdir"
|
|
# shellcheck disable=SC2086
|
|
erl -noshell -eval \
|
|
"ok = emqx_conf:dump_schema('$docdir', $SCHEMA_MODULE), \
|
|
halt(0)."
|
|
}
|
|
|
|
## arg1 is the profile for which the following args (as app names) should be excluded
|
|
assert_no_excluded_deps() {
|
|
local profile="$1"
|
|
shift 1
|
|
if [ "$PROFILE" != "$profile" ]; then
|
|
# not currently building the profile which has apps to be excluded
|
|
return 0
|
|
fi
|
|
local rel_dir="_build/$PROFILE/rel/emqx/lib"
|
|
local excluded_apps=( "$@" )
|
|
local found
|
|
for app in "${excluded_apps[@]}"; do
|
|
found="$($FIND "$rel_dir" -maxdepth 1 -type d -name "$app-*")"
|
|
if [ -n "${found}" ]; then
|
|
echo "ERROR: ${app} should not be included in ${PROFILE}"
|
|
echo "ERROR: found ${app} in ${rel_dir}"
|
|
exit 1
|
|
fi
|
|
done
|
|
}
|
|
|
|
just_compile() {
|
|
./scripts/pre-compile.sh "$PROFILE"
|
|
# make_elixir_rel always create rebar.lock
|
|
# delete it to make git clone + checkout work because we use shallow close for rebar deps
|
|
rm -f rebar.lock
|
|
# compile all beams
|
|
./rebar3 as "$PROFILE" compile
|
|
make_docs
|
|
}
|
|
|
|
just_compile_elixir() {
|
|
./scripts/pre-compile.sh "$PROFILE"
|
|
rm -f rebar.lock
|
|
# shellcheck disable=SC1010
|
|
env MIX_ENV="$PROFILE" mix do local.hex --if-missing --force, \
|
|
local.rebar rebar3 "${PWD}/rebar3" --if-missing --force, \
|
|
deps.get
|
|
env MIX_ENV="$PROFILE" mix compile
|
|
}
|
|
|
|
make_rel() {
|
|
local release_or_tar="${1}"
|
|
just_compile
|
|
# now assemble the release tar
|
|
./rebar3 as "$PROFILE" "$release_or_tar"
|
|
assert_no_excluded_deps emqx-enterprise emqx_telemetry
|
|
}
|
|
|
|
make_elixir_rel() {
|
|
./scripts/pre-compile.sh "$PROFILE"
|
|
export_elixir_release_vars "$PROFILE"
|
|
# for some reason, this has to be run outside "do"...
|
|
mix local.rebar --if-missing --force
|
|
# shellcheck disable=SC1010
|
|
mix do local.hex --if-missing --force, \
|
|
local.rebar rebar3 "${PWD}/rebar3" --if-missing --force, \
|
|
deps.get
|
|
mix release --overwrite
|
|
assert_no_excluded_deps emqx-enterprise emqx_telemetry
|
|
}
|
|
|
|
## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup
|
|
make_relup() {
|
|
local rel_dir="_build/$PROFILE/rel/emqx"
|
|
local name_pattern
|
|
name_pattern="${PROFILE}-$(./pkg-vsn.sh "$PROFILE" --vsn_matcher --long)"
|
|
local releases=()
|
|
mkdir -p _upgrade_base
|
|
while read -r tgzfile ; do
|
|
local base_vsn
|
|
base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9])?(-[0-9a-f]{8})?" | head -1)"
|
|
## we have to create tmp dir to untar old tgz, as `tar --skip-old-files` is not supported on all plantforms
|
|
local tmp_dir
|
|
tmp_dir="$(mktemp -d -t emqx.XXXXXXX)"
|
|
$TAR -C "$tmp_dir" -zxf "$tgzfile"
|
|
mkdir -p "${rel_dir}/releases/"
|
|
cp -npr "$tmp_dir/releases"/* "${rel_dir}/releases/"
|
|
## There is for some reason a copy of the '$PROFILE.rel' file to releases dir,
|
|
## the content is duplicated to releases/5.0.0/$PROFILE.rel.
|
|
## This file seems to be useless, but yet confusing as it does not change after upgrade/downgrade
|
|
## Hence we force delete this file.
|
|
rm -f "${rel_dir}/releases/${PROFILE}.rel"
|
|
mkdir -p "${rel_dir}/lib/"
|
|
cp -npr "$tmp_dir/lib"/* "${rel_dir}/lib/"
|
|
rm -rf "$tmp_dir"
|
|
releases+=( "$base_vsn" )
|
|
done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.tar.gz" -type f)
|
|
if [ ${#releases[@]} -eq 0 ]; then
|
|
log "No upgrade base found, relup ignored"
|
|
return 0
|
|
fi
|
|
RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")"
|
|
export RELX_BASE_VERSIONS
|
|
./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}"
|
|
}
|
|
|
|
cp_dyn_libs() {
|
|
local rel_dir="$1"
|
|
local target_dir="${rel_dir}/dynlibs"
|
|
if ! [ "$(uname -s)" = 'Linux' ]; then
|
|
return 0;
|
|
fi
|
|
mkdir -p "$target_dir"
|
|
while read -r so_file; do
|
|
cp -L "$so_file" "$target_dir/"
|
|
done < <("$FIND" "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \
|
|
| xargs -0 ldd \
|
|
| grep -E '(libcrypto)|(libtinfo)|(libatomic)' \
|
|
| awk '{print $3}' \
|
|
| sort -u)
|
|
}
|
|
|
|
## Re-pack the relx assembled .tar.gz to EMQX's package naming scheme
|
|
## It assumes the .tar.gz has been built -- relies on Makefile dependency
|
|
make_tgz() {
|
|
local pkgpath="_packages/${PROFILE}"
|
|
local src_tarball
|
|
local target_name
|
|
local target
|
|
|
|
if [ "${IS_ELIXIR:-no}" = "yes" ]
|
|
then
|
|
# ensure src_tarball exists
|
|
ELIXIR_MAKE_TAR=yes make_elixir_rel
|
|
|
|
local relpath="_build/${PROFILE}"
|
|
full_vsn="$(./pkg-vsn.sh "$PROFILE" --long --elixir)"
|
|
else
|
|
# build the src_tarball again to ensure relup is included
|
|
# elixir does not have relup yet.
|
|
make_rel tar
|
|
|
|
local relpath="_build/${PROFILE}/rel/emqx"
|
|
full_vsn="$(./pkg-vsn.sh "$PROFILE" --long)"
|
|
fi
|
|
|
|
case "$SYSTEM" in
|
|
macos*)
|
|
target_name="${PROFILE}-${full_vsn}.zip"
|
|
;;
|
|
windows*)
|
|
target_name="${PROFILE}-${full_vsn}.zip"
|
|
;;
|
|
*)
|
|
target_name="${PROFILE}-${full_vsn}.tar.gz"
|
|
;;
|
|
esac
|
|
|
|
target="${pkgpath}/${target_name}"
|
|
|
|
src_tarball="${relpath}/emqx-${PKG_VSN}.tar.gz"
|
|
tard="$(mktemp -d -t emqx.XXXXXXX)"
|
|
mkdir -p "${tard}/emqx"
|
|
mkdir -p "${pkgpath}"
|
|
if [ ! -f "$src_tarball" ]; then
|
|
log "ERROR: $src_tarball is not found"
|
|
fi
|
|
$TAR zxf "${src_tarball}" -C "${tard}/emqx"
|
|
if [ -f "${tard}/emqx/releases/${PKG_VSN}/relup" ]; then
|
|
./scripts/relup-build/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup"
|
|
fi
|
|
## try to be portable for tar.gz packages.
|
|
## for DEB and RPM packages the dependencies are resoved by yum and apt
|
|
cp_dyn_libs "${tard}/emqx"
|
|
case "$SYSTEM" in
|
|
macos*)
|
|
# if the flag to sign macos binaries is set, but developer certificate
|
|
# or certificate password is not configured, reset the flag
|
|
# could happen, for example, when people submit PR from a fork, in this
|
|
# case they cannot access secrets
|
|
if [[ "${APPLE_SIGN_BINARIES:-0}" == 1 && \
|
|
( "${APPLE_DEVELOPER_ID_BUNDLE:-0}" == 0 || \
|
|
"${APPLE_DEVELOPER_ID_BUNDLE_PASSWORD:-0}" == 0 ) ]]; then
|
|
echo "Apple developer certificate is not configured, skip signing"
|
|
APPLE_SIGN_BINARIES=0
|
|
fi
|
|
if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then
|
|
./scripts/macos-sign-binaries.sh "${tard}/emqx"
|
|
fi
|
|
## create zip after change dir
|
|
## to avoid creating an extra level of 'emqx' dir in the .zip file
|
|
pushd "${tard}/emqx" >/dev/null
|
|
zip -r "../${target_name}" -- * >/dev/null
|
|
popd >/dev/null
|
|
mv "${tard}/${target_name}" "${target}"
|
|
if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then
|
|
# notarize the package
|
|
# if fails, check what went wrong with this command:
|
|
# xcrun notarytool log --apple-id <apple id> \
|
|
# --apple-id <apple id> \
|
|
# --password <apple id password>
|
|
# --team-id <apple team id> <submission-id>
|
|
echo 'Submitting the package for notarization to Apple (normally takes about a minute)'
|
|
notarytool_output="$(xcrun notarytool submit \
|
|
--apple-id "${APPLE_ID}" \
|
|
--password "${APPLE_ID_PASSWORD}" \
|
|
--team-id "${APPLE_TEAM_ID}" "${target}" \
|
|
--no-progress \
|
|
--wait)"
|
|
echo "$notarytool_output"
|
|
echo "$notarytool_output" | grep -q 'status: Accepted' || {
|
|
echo 'Notarization failed';
|
|
exit 1;
|
|
}
|
|
fi
|
|
# sha256sum may not be available on macos
|
|
openssl dgst -sha256 "${target}" | cut -d ' ' -f 2 > "${target}.sha256"
|
|
;;
|
|
windows*)
|
|
pushd "${tard}" >/dev/null
|
|
7z a "${target_name}" ./emqx/* >/dev/null
|
|
popd >/dev/null
|
|
mv "${tard}/${target_name}" "${target}"
|
|
sha256sum "${target}" | head -c 64 > "${target}.sha256"
|
|
;;
|
|
*)
|
|
## create tar after change dir
|
|
## to avoid creating an extra level of 'emqx' dir in the .tar.gz file
|
|
pushd "${tard}/emqx" >/dev/null
|
|
$TAR -zcf "../${target_name}" -- *
|
|
popd >/dev/null
|
|
mv "${tard}/${target_name}" "${target}"
|
|
sha256sum "${target}" | head -c 64 > "${target}.sha256"
|
|
;;
|
|
esac
|
|
log "Archive successfully repacked: ${target}"
|
|
log "Archive sha256sum: $(cat "${target}.sha256")"
|
|
}
|
|
|
|
trap docker_cleanup EXIT
|
|
|
|
docker_cleanup() {
|
|
rm -f ./.dockerignore >/dev/null
|
|
}
|
|
|
|
## This function builds the default docker image based on debian 11
|
|
make_docker() {
|
|
EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}"
|
|
EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
|
|
EMQX_DOCKERFILE="${EMQX_DOCKERFILE:-deploy/docker/Dockerfile}"
|
|
if [[ "$PROFILE" = *-elixir ]]; then
|
|
PKG_VSN="$PKG_VSN-elixir"
|
|
fi
|
|
local default_tag="emqx/${PROFILE%%-elixir}:${PKG_VSN}"
|
|
EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-$default_tag}"
|
|
## extra_deps is a comma separated list of debian 11 package names
|
|
local extra_deps=''
|
|
if [[ "$PROFILE" = *enterprise* ]]; then
|
|
extra_deps='libsasl2-2'
|
|
fi
|
|
echo '_build' >> ./.dockerignore
|
|
set -x
|
|
docker build --no-cache --pull \
|
|
--build-arg BUILD_FROM="${EMQX_BUILDER}" \
|
|
--build-arg RUN_FROM="${EMQX_RUNNER}" \
|
|
--build-arg EMQX_NAME="${PROFILE}" \
|
|
--build-arg EXTRA_DEPS="${extra_deps}" \
|
|
--tag "${EMQX_IMAGE_TAG}" \
|
|
-f "${EMQX_DOCKERFILE}" .
|
|
[[ "${DEBUG:-}" -eq 1 ]] || set +x
|
|
}
|
|
|
|
function join {
|
|
local IFS="$1"
|
|
shift
|
|
echo "$*"
|
|
}
|
|
|
|
# used to control the Elixir Mix Release output
|
|
# see docstring in `mix.exs`
|
|
export_elixir_release_vars() {
|
|
local profile="$1"
|
|
case "$profile" in
|
|
emqx|emqx-enterprise)
|
|
export ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no}
|
|
;;
|
|
emqx-pkg|emqx-enterprise-pkg)
|
|
export ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes}
|
|
;;
|
|
*)
|
|
echo Invalid profile "$profile"
|
|
exit 1
|
|
esac
|
|
export MIX_ENV="$profile"
|
|
}
|
|
|
|
log "building artifact=$ARTIFACT for profile=$PROFILE"
|
|
|
|
case "$ARTIFACT" in
|
|
apps)
|
|
if [ "${IS_ELIXIR:-}" = "yes" ]; then
|
|
just_compile_elixir
|
|
else
|
|
just_compile
|
|
fi
|
|
;;
|
|
doc|docs)
|
|
make_docs
|
|
;;
|
|
rel)
|
|
make_rel release
|
|
;;
|
|
relup)
|
|
make_relup
|
|
;;
|
|
tgz)
|
|
make_tgz
|
|
;;
|
|
pkg)
|
|
# this only affect build artifacts, such as schema doc
|
|
export EMQX_ETC_DIR='/etc/emqx/'
|
|
if [ -z "${PKGERDIR:-}" ]; then
|
|
log "Skipped making deb/rpm package for $SYSTEM"
|
|
exit 0
|
|
fi
|
|
export EMQX_REL_FORM="$PKGERDIR"
|
|
if [ "${IS_ELIXIR:-}" = 'yes' ]; then
|
|
make_elixir_rel
|
|
else
|
|
make_rel tar
|
|
fi
|
|
env EMQX_REL="$(pwd)" \
|
|
EMQX_BUILD="${PROFILE}" \
|
|
make -C "deploy/packages/${PKGERDIR}" clean
|
|
env EMQX_REL="$(pwd)" \
|
|
EMQX_BUILD="${PROFILE}" \
|
|
make -C "deploy/packages/${PKGERDIR}"
|
|
;;
|
|
docker)
|
|
make_docker
|
|
;;
|
|
elixir)
|
|
make_elixir_rel
|
|
;;
|
|
*)
|
|
log "Unknown artifact $ARTIFACT"
|
|
exit 1
|
|
;;
|
|
esac
|