#!/usr/bin/env bash # This script helps to build release artifacts. # arg1: profile, e.g. emqx | emqx-enterprise # arg2: artifact, e.g. rel | relup | tgz | pkg set -euo pipefail if [ "${DEBUG:-0}" -eq 1 ]; then set -x # set this for rebar3 export DIAGNOSTIC=1 fi log_red() { local RED='\033[0;31m' # Red local NC='\033[0m' # No Color echo -e "${RED}${1}${NC}" } PROFILE_ARG="$1" ARTIFACT="$2" is_enterprise() { case "$1" in *enterprise*) echo 'yes' ;; *) echo 'no' ;; esac } PROFILE_ENV="${PROFILE:-${PROFILE_ARG}}" case "$(is_enterprise "$PROFILE_ARG"),$(is_enterprise "$PROFILE_ENV")" in 'yes,yes') true ;; 'no,no') true ;; *) log_red "PROFILE env var is set to '$PROFILE_ENV', but '$0' arg1 is '$PROFILE_ARG'" exit 1 ;; esac # make sure PROFILE is exported, it is needed by rebar.config.erl PROFILE=$PROFILE_ARG export PROFILE # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" export PKG_VSN SYSTEM="$(./scripts/get-distro.sh)" ARCH="$(uname -m)" case "$ARCH" in x86_64) ARCH='amd64' ;; aarch64) ARCH='arm64' ;; arm*) ARCH='arm64' ;; esac export ARCH ## ## Support RPM and Debian based linux systems ## if [ "$(uname -s)" = 'Linux' ]; then case "${SYSTEM:-}" in ubuntu*|debian*|raspbian*) PKGERDIR='deb' ;; *) PKGERDIR='rpm' ;; esac fi if [ "${SYSTEM}" = 'windows' ]; then # windows does not like the find FIND="/usr/bin/find" TAR="/usr/bin/tar" export BUILD_WITHOUT_ROCKSDB="on" else FIND='find' TAR='tar' fi log() { local msg="$1" # rebar3 prints ===>, so we print ===< echo "===< $msg" } prepare_erl_libs() { local libs_dir="$1" local erl_libs="${ERL_LIBS:-}" local sep if [ "${SYSTEM}" = 'windows' ]; then sep=';' else sep=':' fi for app in "${libs_dir}"/*; do if [ -d "${app}/ebin" ]; then if [ -n "$erl_libs" ]; then erl_libs="${erl_libs}${sep}${app}" else erl_libs="${app}" fi fi done export ERL_LIBS="$erl_libs" } make_docs() { case "$(is_enterprise "$PROFILE")" in 'yes') SCHEMA_MODULE='emqx_enterprise_schema' ;; 'no') SCHEMA_MODULE='emqx_conf_schema' ;; esac prepare_erl_libs "_build/$PROFILE/checkouts" prepare_erl_libs "_build/$PROFILE/lib" local docdir="_build/docgen/$PROFILE" mkdir -p "$docdir" # shellcheck disable=SC2086 erl -enable-feature maybe_expr -noshell -eval \ "ok = emqx_conf:dump_schema('$docdir', $SCHEMA_MODULE), \ halt(0)." local desc="$docdir/desc.en.hocon" if command -v jq &> /dev/null; then log "Generating $desc" scripts/merge-i18n.escript | jq --sort-keys . > "$desc" else # it is not a big deal if we cannot generate the desc log_red "NOT Generated: $desc" fi } ## arg1 is the profile for which the following args (as app names) should be excluded assert_no_excluded_deps() { local profile="$1" shift 1 if [ "$PROFILE" != "$profile" ]; then # not currently building the profile which has apps to be excluded return 0 fi local rel_dir="_build/$PROFILE/rel/emqx/lib" local excluded_apps=( "$@" ) local found for app in "${excluded_apps[@]}"; do found="$($FIND "$rel_dir" -maxdepth 1 -type d -name "$app-*")" if [ -n "${found}" ]; then log_red "ERROR: ${app} should not be included in ${PROFILE}" log_red "ERROR: found ${app} in ${rel_dir}" exit 1 fi done } just_compile() { ./scripts/pre-compile.sh "$PROFILE" # make_elixir_rel always create rebar.lock # delete it to make git clone + checkout work because we use shallow close for rebar deps rm -f rebar.lock # compile all beams ./rebar3 as "$PROFILE" compile make_docs } just_compile_elixir() { ./scripts/pre-compile.sh "$PROFILE" rm -f rebar.lock env MIX_ENV="$PROFILE" mix local.rebar --if-missing --force env MIX_ENV="$PROFILE" mix local.rebar rebar3 "${PWD}/rebar3" --if-missing --force env MIX_ENV="$PROFILE" mix local.hex --if-missing --force env MIX_ENV="$PROFILE" mix deps.get env MIX_ENV="$PROFILE" mix compile } make_rel() { local release_or_tar="${1}" just_compile # now assemble the release tar ./rebar3 as "$PROFILE" "$release_or_tar" assert_no_excluded_deps emqx-enterprise emqx_telemetry } make_elixir_rel() { ./scripts/pre-compile.sh "$PROFILE" export_elixir_release_vars "$PROFILE" env MIX_ENV="$PROFILE" mix local.rebar --if-missing --force env MIX_ENV="$PROFILE" mix local.rebar rebar3 "${PWD}/rebar3" --if-missing --force env MIX_ENV="$PROFILE" mix local.hex --if-missing --force env MIX_ENV="$PROFILE" mix deps.get env MIX_ENV="$PROFILE" mix release --overwrite assert_no_excluded_deps emqx-enterprise emqx_telemetry } ## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup make_relup() { local rel_dir="_build/$PROFILE/rel/emqx" local name_pattern name_pattern="${PROFILE}-$(./pkg-vsn.sh "$PROFILE" --vsn_matcher --long)" local releases=() mkdir -p _upgrade_base while read -r tgzfile ; do local base_vsn base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9])?(-[0-9a-f]{8})?" | head -1)" ## we have to create tmp dir to untar old tgz, as `tar --skip-old-files` is not supported on all plantforms local tmp_dir tmp_dir="$(mktemp -d -t emqx.XXXXXXX)" $TAR -C "$tmp_dir" -zxf "$tgzfile" mkdir -p "${rel_dir}/releases/" cp -npr "$tmp_dir/releases"/* "${rel_dir}/releases/" ## There is for some reason a copy of the '$PROFILE.rel' file to releases dir, ## the content is duplicated to releases/5.0.0/$PROFILE.rel. ## This file seems to be useless, but yet confusing as it does not change after upgrade/downgrade ## Hence we force delete this file. rm -f "${rel_dir}/releases/${PROFILE}.rel" mkdir -p "${rel_dir}/lib/" cp -npr "$tmp_dir/lib"/* "${rel_dir}/lib/" rm -rf "$tmp_dir" releases+=( "$base_vsn" ) done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.tar.gz" -type f) if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" return 0 fi RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")" export RELX_BASE_VERSIONS ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}" } cp_dyn_libs() { local rel_dir="$1" local target_dir="${rel_dir}/dynlibs" if ! [ "$(uname -s)" = 'Linux' ]; then return 0; fi mkdir -p "$target_dir" while read -r so_file; do cp -L "$so_file" "$target_dir/" done < <("$FIND" "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \ | xargs -0 ldd \ | grep -E '(libcrypto)|(libtinfo)|(libatomic)' \ | awk '{print $3}' \ | sort -u) } ## Re-pack the relx assembled .tar.gz to EMQX's package naming scheme ## It assumes the .tar.gz has been built -- relies on Makefile dependency make_tgz() { local pkgpath="_packages/${PROFILE}" local src_tarball local target_name local target if [ "${IS_ELIXIR:-no}" = "yes" ] then # ensure src_tarball exists ELIXIR_MAKE_TAR=yes make_elixir_rel local relpath="_build/${PROFILE}" full_vsn="$(./pkg-vsn.sh "$PROFILE" --long --elixir)" else # build the src_tarball again to ensure relup is included # elixir does not have relup yet. make_rel tar local relpath="_build/${PROFILE}/rel/emqx" full_vsn="$(./pkg-vsn.sh "$PROFILE" --long)" fi case "$SYSTEM" in macos*) target_name="${PROFILE}-${full_vsn}.zip" ;; windows*) target_name="${PROFILE}-${full_vsn}.zip" ;; *) target_name="${PROFILE}-${full_vsn}.tar.gz" ;; esac target="${pkgpath}/${target_name}" src_tarball="${relpath}/emqx-${PKG_VSN}.tar.gz" tard="$(mktemp -d -t emqx.XXXXXXX)" mkdir -p "${tard}/emqx" mkdir -p "${pkgpath}" if [ ! -f "$src_tarball" ]; then log_red "ERROR: $src_tarball is not found" fi $TAR zxf "${src_tarball}" -C "${tard}/emqx" if [ -f "${tard}/emqx/releases/${PKG_VSN}/relup" ]; then ./scripts/relup-build/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup" fi ## try to be portable for tar.gz packages. ## for DEB and RPM packages the dependencies are resoved by yum and apt cp_dyn_libs "${tard}/emqx" case "$SYSTEM" in macos*) # if the flag to sign macos binaries is set, but developer certificate # or certificate password is not configured, reset the flag # could happen, for example, when people submit PR from a fork, in this # case they cannot access secrets if [[ "${APPLE_SIGN_BINARIES:-0}" == 1 && \ ( "${APPLE_DEVELOPER_ID_BUNDLE:-0}" == 0 || \ "${APPLE_DEVELOPER_ID_BUNDLE_PASSWORD:-0}" == 0 ) ]]; then echo "Apple developer certificate is not configured, skip signing" APPLE_SIGN_BINARIES=0 fi if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then ./scripts/macos-sign-binaries.sh "${tard}/emqx" fi ## create zip after change dir ## to avoid creating an extra level of 'emqx' dir in the .zip file pushd "${tard}/emqx" >/dev/null zip -r "../${target_name}" -- * >/dev/null popd >/dev/null mv "${tard}/${target_name}" "${target}" if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then # notarize the package # if fails, check what went wrong with this command: # xcrun notarytool log \ # --apple-id \ # --password # --team-id echo 'Submitting the package for notarization to Apple (normally takes about a minute)' notarytool_output="$(xcrun notarytool submit \ --apple-id "${APPLE_ID}" \ --password "${APPLE_ID_PASSWORD}" \ --team-id "${APPLE_TEAM_ID}" "${target}" \ --no-progress \ --wait)" echo "$notarytool_output" echo "$notarytool_output" | grep -q 'status: Accepted' || { echo 'Notarization failed'; exit 1; } fi # sha256sum may not be available on macos openssl dgst -sha256 "${target}" | cut -d ' ' -f 2 > "${target}.sha256" ;; windows*) pushd "${tard}" >/dev/null 7z a "${target_name}" ./emqx/* >/dev/null popd >/dev/null mv "${tard}/${target_name}" "${target}" sha256sum "${target}" | head -c 64 > "${target}.sha256" ;; *) ## create tar after change dir ## to avoid creating an extra level of 'emqx' dir in the .tar.gz file pushd "${tard}/emqx" >/dev/null $TAR -zcf "../${target_name}" -- * popd >/dev/null mv "${tard}/${target_name}" "${target}" sha256sum "${target}" | head -c 64 > "${target}.sha256" ;; esac log "Archive successfully repacked: ${target}" log "Archive sha256sum: $(cat "${target}.sha256")" } docker_cleanup() { rm -f ./.dockerignore >/dev/null # shellcheck disable=SC2015 [ -f ./.dockerignore.bak ] && mv ./.dockerignore.bak ./.dockerignore >/dev/null || true } function is_ecr_and_enterprise() { local registry="$1" local profile="$2" if [[ "$registry" == public.ecr.aws* ]] && [[ "$profile" == *enterprise* ]]; then return 0 else return 1 fi } ## Build the default docker image based on debian 12. make_docker() { local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.3-2}" local EMQX_BUILDER_PLATFORM="${EMQX_BUILDER_PLATFORM:-debian12}" local EMQX_BUILDER_OTP="${EMQX_BUILDER_OTP:-25.3.2-2}" local EMQX_BUILDER_ELIXIR="${EMQX_BUILDER_ELIXIR:-1.15.7}" local EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${EMQX_BUILDER_ELIXIR}-${EMQX_BUILDER_OTP}-${EMQX_BUILDER_PLATFORM}} local EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}" local EMQX_DOCKERFILE="${EMQX_DOCKERFILE:-deploy/docker/Dockerfile}" local EMQX_SOURCE_TYPE="${EMQX_SOURCE_TYPE:-src}" # shellcheck disable=SC2155 local VSN_MAJOR="$(scripts/semver.sh "$PKG_VSN" --major)" # shellcheck disable=SC2155 local VSN_MINOR="$(scripts/semver.sh "$PKG_VSN" --minor)" # shellcheck disable=SC2155 local VSN_PATCH="$(scripts/semver.sh "$PKG_VSN" --patch)" local SUFFIX='' if [[ "$PROFILE" = *-elixir ]]; then SUFFIX="-elixir" fi local DOCKER_REGISTRY="${DOCKER_REGISTRY:-docker.io}" local DOCKER_REGISTRIES=( ) IFS=',' read -ra DOCKER_REGISTRY_ARR <<< "$DOCKER_REGISTRY" for r in "${DOCKER_REGISTRY_ARR[@]}"; do # append to DOCKER_REGISTRIES DOCKER_REGISTRIES+=("$r") done local DOCKER_ORG="${DOCKER_ORG:-emqx}" local EMQX_BASE_DOCKER_TAG="${DOCKER_ORG}/${PROFILE%%-elixir}" local default_tag="${EMQX_BASE_DOCKER_TAG}:${PKG_VSN}${SUFFIX}" local EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-$default_tag}" local EDITION=Opensource local LICENSE='Apache-2.0' local PRODUCT_URL='https://www.emqx.io' local PRODUCT_DESCRIPTION='Official docker image for EMQX, the most scalable open-source MQTT broker for IoT, IIoT, and connected vehicles.' local DOCUMENTATION_URL='https://www.emqx.io/docs/en/latest/' ## extra_deps is a comma separated list of debian 12 package names local EXTRA_DEPS='' if [[ "$PROFILE" = *enterprise* ]]; then EXTRA_DEPS='libsasl2-2,libsasl2-modules-gssapi-mit' EDITION=Enterprise LICENSE='(Apache-2.0 AND BSL-1.1)' PRODUCT_URL='https://www.emqx.com/en/products/emqx' PRODUCT_DESCRIPTION='Official docker image for EMQX Enterprise, an enterprise MQTT platform at scale. ' DOCUMENTATION_URL='https://docs.emqx.com/en/enterprise/latest/' fi local ISO_8601_DATE GIT_REVISION ISO_8601_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" GIT_REVISION="$(git rev-parse HEAD)" export BUILDX_NO_DEFAULT_ATTESTATIONS=1 local DOCKER_BUILDX_ARGS=( --build-arg BUILD_FROM="${EMQX_BUILDER}" \ --build-arg RUN_FROM="${EMQX_RUNNER}" \ --build-arg SOURCE_TYPE="${EMQX_SOURCE_TYPE}" \ --build-arg PROFILE="${PROFILE%%-elixir}" \ --build-arg IS_ELIXIR="$([[ "$PROFILE" = *-elixir ]] && echo yes || echo no)" \ --build-arg SUFFIX="${SUFFIX}" \ --build-arg EXTRA_DEPS="${EXTRA_DEPS}" \ --build-arg PKG_VSN="${PKG_VSN}" \ --file "${EMQX_DOCKERFILE}" \ --label org.opencontainers.image.title="${PROFILE%%-elixir}" \ --label org.opencontainers.image.edition="${EDITION}" \ --label org.opencontainers.image.version="${PKG_VSN}" \ --label org.opencontainers.image.revision="${GIT_REVISION}" \ --label org.opencontainers.image.created="${ISO_8601_DATE}" \ --label org.opencontainers.image.source='https://github.com/emqx/emqx' \ --label org.opencontainers.image.url="${PRODUCT_URL}" \ --label org.opencontainers.image.description="${PRODUCT_DESCRIPTION}" \ --label org.opencontainers.image.documentation="${DOCUMENTATION_URL}" \ --label org.opencontainers.image.licenses="${LICENSE}" \ --label org.opencontainers.image.otp.version="${EMQX_BUILDER_OTP}" \ --pull ) :> ./.emqx_docker_image_tags for r in "${DOCKER_REGISTRIES[@]}"; do DOCKER_BUILDX_ARGS+=(--tag "$r/${EMQX_IMAGE_TAG}") echo "$r/${EMQX_IMAGE_TAG}" >> ./.emqx_docker_image_tags done if [ "${DOCKER_BUILD_NOCACHE:-false}" = true ]; then DOCKER_BUILDX_ARGS+=(--no-cache) fi if [ "${SUFFIX}" = '-elixir' ]; then DOCKER_BUILDX_ARGS+=(--label org.opencontainers.image.elixir.version="${EMQX_BUILDER_ELIXIR}") fi if [ "${DOCKER_LATEST:-false}" = true ]; then for r in "${DOCKER_REGISTRIES[@]}"; do DOCKER_BUILDX_ARGS+=(--tag "$r/${EMQX_BASE_DOCKER_TAG}:latest${SUFFIX}") echo "$r/${EMQX_BASE_DOCKER_TAG}:latest${SUFFIX}" >> ./.emqx_docker_image_tags DOCKER_BUILDX_ARGS+=(--tag "$r/${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}${SUFFIX}") echo "$r/${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}${SUFFIX}" >> ./.emqx_docker_image_tags DOCKER_BUILDX_ARGS+=(--tag "$r/${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}.${VSN_PATCH}${SUFFIX}") echo "$r/${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}.${VSN_PATCH}${SUFFIX}" >> ./.emqx_docker_image_tags done fi if [ "${DOCKER_PLATFORMS:-default}" != 'default' ]; then DOCKER_BUILDX_ARGS+=(--platform "${DOCKER_PLATFORMS}") fi if [ "${DOCKER_PUSH:-false}" = true ]; then DOCKER_BUILDX_ARGS+=(--push) fi if [ "${DOCKER_LOAD:-false}" = true ]; then DOCKER_BUILDX_ARGS+=(--load) fi if [ -d "${REBAR_GIT_CACHE_DIR:-}" ]; then cache_tar="$(pwd)/rebar-git-cache.tar" if [ ! -f "${cache_tar}" ]; then pushd "${REBAR_GIT_CACHE_DIR}" >/dev/null tar -cf "${cache_tar}" . popd >/dev/null fi fi if [ -n "${DEBUG:-}" ]; then DOCKER_BUILDX_ARGS+=(--build-arg DEBUG="${DEBUG}" --progress=plain) fi # shellcheck disable=SC2015 [ -f ./.dockerignore ] && mv ./.dockerignore ./.dockerignore.bak || true trap docker_cleanup EXIT { echo '_build/' echo 'deps/' echo '*.lock' echo '_packages/' echo '.vs/' echo '.vscode/' echo 'lux_logs/' echo '_upgrade_base/' } >> ./.dockerignore echo "Docker buildx args: ${DOCKER_BUILDX_ARGS[*]}" docker buildx build "${DOCKER_BUILDX_ARGS[@]}" . } function join { local IFS="$1" shift echo "$*" } # used to control the Elixir Mix Release output # see docstring in `mix.exs` export_elixir_release_vars() { local profile="$1" case "$profile" in emqx|emqx-enterprise) export ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no} ;; emqx-pkg|emqx-enterprise-pkg) export ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes} ;; *) echo Invalid profile "$profile" exit 1 esac export MIX_ENV="$profile" } log "building artifact=$ARTIFACT for profile=$PROFILE" case "$ARTIFACT" in apps) if [ "${IS_ELIXIR:-}" = "yes" ]; then just_compile_elixir else just_compile fi ;; doc|docs) make_docs ;; rel) make_rel release ;; relup) make_relup ;; tgz) make_tgz ;; pkg) # this only affect build artifacts, such as schema doc export EMQX_ETC_DIR='/etc/emqx/' if [ -z "${PKGERDIR:-}" ]; then log "Skipped making deb/rpm package for $SYSTEM" exit 0 fi export EMQX_REL_FORM="$PKGERDIR" if [ "${IS_ELIXIR:-}" = 'yes' ]; then make_elixir_rel else make_rel tar fi env EMQX_REL="$(pwd)" \ EMQX_BUILD="${PROFILE}" \ make -C "deploy/packages/${PKGERDIR}" clean env EMQX_REL="$(pwd)" \ EMQX_BUILD="${PROFILE}" \ make -C "deploy/packages/${PKGERDIR}" ;; docker) make_docker ;; elixir) make_elixir_rel ;; *) log "Unknown artifact $ARTIFACT" exit 1 ;; esac