diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 8735e0d04..84e02bf89 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -227,6 +227,9 @@ jobs: - centos6 # - raspbian10 #armv6l is too slow to emulate # - raspbian9 + otp_version: + #- 23.2.7.2-emqx-3 + - 23.3.4.9-3 exclude: - os: centos6 arch: arm64 @@ -265,7 +268,7 @@ jobs: run: unzip -q source.zip - name: build emqx packages env: - ERL_OTP: erl23.2.7.2-emqx-3 + ERL_OTP: erl${{ matrix.otp_version }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} @@ -289,11 +292,21 @@ jobs: docker volume prune -f - name: create sha256 env: - PROFILE: ${{ matrix.profile}} + PROFILE: ${{ matrix.profile }} + ERL_OTP: erl${{ matrix.otp_version }} + ARCH: ${{ matrix.arch }} run: | if [ -d /tmp/packages/$PROFILE ]; then cd /tmp/packages/$PROFILE for var in $(ls emqx-* ); do + if [[ $ERL_OTP == erl23.2* ]]; then + # Keep package with new OTP as default + # But move package with old otp to track 2 + echo "rename track 2 package" + oldfile="$var" + var="${var/${ARCH}/otp23.2-${ARCH}}" + mv "$oldfile" "$var" + fi bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" done cd - @@ -372,7 +385,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine + BUILD_FROM=emqx/build-env:erl23.3.4.9-3-alpine RUN_FROM=alpine:3.12 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -388,7 +401,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine + BUILD_FROM=emqx/build-env:erl23.3.4.9-3-alpine RUN_FROM=alpine:3.12 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise diff --git a/bin/emqx b/bin/emqx index ec5de13b8..20cf266b0 100755 --- a/bin/emqx +++ b/bin/emqx @@ -43,6 +43,8 @@ assert_node_alive() { } check_erlang_start() { + # Fix bin permission + find "$BINDIR" ! -executable -exec chmod a+x {} \; "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()" } diff --git a/build b/build index 44287bff5..2f773d295 100755 --- a/build +++ b/build @@ -94,7 +94,37 @@ make_relup() { fi RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")" export RELX_BASE_VERSIONS + if [[ ${PKG_VSN} == 4.3* ]]; then + echo "EMQX 4.3 specific, overwrite OTP app versions" + local emqx_rel_file="${releases_dir}/${PKG_VSN}/emqx.rel" + if [ ! -f "${emqx_rel_file}" ]; then + ./rebar3 as "${PROFILE}" release + fi + scripts/emqx_rel_otp_app_overwrite.escript "${releases_dir}" "${PROFILE}" "${PKG_VSN}" "${RELX_BASE_VERSIONS}" + fi ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}" + + # assert that there is no 'restart_emulator' in relup + # EMQX wants hot upgrade without VM restart + if grep restart_emulator "${releases_dir}/${PKG_VSN}/relup"; then + echo "Error: restart_emulator instruction found in relup"; + exit 1 + fi + + # rollback rel file per releases + # + if [[ ${PKG_VSN} == 4.3* ]]; then + echo "restore upgrade base rel files... " + for rel in ${releases[*]}; + do + bakfile="${releases_dir}/${rel}/${PROFILE}.rel.bak" + echo "restore $bakfile ..." + if [ -f "$bakfile" ]; then + echo "restore from $bakfile" + mv "${bakfile}" "${bakfile%.bak}" + fi + done + fi } cp_dyn_libs() { diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index f24b4a348..346d33630 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine +ARG BUILD_FROM=emqx/build-env:erl23.3.4.9-3-alpine ARG RUN_FROM=alpine:3.12 FROM ${BUILD_FROM} AS builder diff --git a/rebar.config.erl b/rebar.config.erl index 91f5f8058..db761e367 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -246,21 +246,10 @@ overlay_vars_pkg(pkg) -> ]. relx_apps(ReleaseType) -> - [ kernel - , sasl - , crypto - , public_key - , asn1 - , syntax_tools - , ssl - , os_mon - , inets - , compiler - , runtime_tools - , redbug + relx_otp_apps() ++ + [ redbug , cuttlefish , emqx - , {mnesia, load} , {ekka, load} , {emqx_plugin_libs, load} , observer_cli @@ -271,9 +260,13 @@ relx_apps(ReleaseType) -> ++ relx_apps_per_rel(ReleaseType) ++ [{N, load} || N <- relx_plugin_apps(ReleaseType)]. +relx_otp_apps() -> + {ok, [Apps]} = file:consult("scripts/rel_otp_apps.eterm"), + true = is_list(Apps), + Apps. + relx_apps_per_rel(cloud) -> [ luerl - , xmerl | [{observer, load} || is_app(observer)] ]; relx_apps_per_rel(edge) -> diff --git a/scripts/emqx_rel_otp_app_overwrite.escript b/scripts/emqx_rel_otp_app_overwrite.escript new file mode 100755 index 000000000..5eace01ff --- /dev/null +++ b/scripts/emqx_rel_otp_app_overwrite.escript @@ -0,0 +1,47 @@ +#!/usr/bin/env escript +%% This script is part of 'relup' process to overwrite the OTP app versions (incl. ERTS) in rel files from upgrade base +%% so that 'rebar relup' call will not generate instructions for restarting OTP apps or restarting the emulator. +%% +%% It simply read OTP app version (incl. ERTS) from the rel file of *NEW* Release ($RelVsn) and write back to the ones +%% in *OLD* versions ($BASE_VERSIONS) +%% +%% note, we use NEW to overwrite OLD is because the modified NEW rel file will be overwritten by next 'rebar relup' +%% +main([Dir, Profile, RelVsn, BASE_VERSIONS]) -> + {ErtsVsn, Overwrites} = get_otp_apps(rel_file(Profile, Dir, RelVsn), RelVsn), + lists:foreach(fun(BaseVer) -> + base_rel_overwrites(BaseVer, Profile, Dir, ErtsVsn, Overwrites) + end, string:tokens(BASE_VERSIONS, ",")). + +get_otp_apps(RelFile, RelVsn) -> + {ok, [{release, {"emqx", RelVsn}, {erts, ErtsVsn}, AppList}]} = file:consult(RelFile), + Apps = lists:filter(fun(X) -> lists:member(element(1, X), otp_apps()) end, AppList), + {ErtsVsn, Apps}. + +base_rel_overwrites(RelVsn, Profile, Dir, ErtsVsn, Overwrites) -> + RelFile = rel_file(Profile, Dir, RelVsn), + file:copy(RelFile, RelFile++".bak"), + {ok, [{release, {"emqx", RelVsn}, {erts, _BaseErtsVsn}, BaseAppList}]} = file:consult(RelFile), + NewData = [ {release, {"emqx", RelVsn}, {erts, ErtsVsn}, + lists:map(fun(X) -> + Name = element(1, X), + case lists:keyfind(Name, 1, Overwrites) of + false -> X; + Y when is_tuple(Y) -> Y + end + end, BaseAppList) + } + ], + ok = file:write_file(RelFile, io_lib:format("~p.", NewData)). + +rel_file("emqx-edge", Dir, RelVsn)-> + rel_file("emqx", Dir, RelVsn); +rel_file(Profile, Dir, RelVsn)-> + filename:join([Dir, RelVsn, Profile++".rel"]). + +otp_apps() -> + {ok, [Apps]} = file:consult("scripts/rel_otp_apps.eterm"), + true = is_list(Apps), + lists:map(fun(App) when is_atom(App) -> App; + ({App, _}) -> App %% handle like {mnesia, load} + end, Apps). diff --git a/scripts/rel_otp_apps.eterm b/scripts/rel_otp_apps.eterm new file mode 100644 index 000000000..4999bbffa --- /dev/null +++ b/scripts/rel_otp_apps.eterm @@ -0,0 +1,16 @@ +%% Single source of truth of list otp apps that we use +[ kernel +, stdlib +, sasl +, crypto +, public_key +, asn1 +, syntax_tools +, ssl +, os_mon +, inets +, compiler +, runtime_tools +, {mnesia, load} +, xmerl +].