diff --git a/build b/build index a339f803b..8bf435357 100755 --- a/build +++ b/build @@ -18,7 +18,7 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" export PKG_VSN -SYSTEM="$(./scripts/get-distro.sh)" +SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" ## ## Support RPM and Debian based linux systems @@ -136,6 +136,9 @@ make_zip() { local tarball="${relpath}/${tarname}" local target_zip="${pkgpath}/${pkgname}" tar zxf "${tarball}" -C "${tard}/emqx" + if ! [[ $SYSTEM == windows* ]]; then + ./scripts/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup" + fi cp_dyn_libs "${tard}/emqx" pushd "${tard}" >/dev/null case "$SYSTEM" in diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src index 902585ffb..513757418 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src @@ -1,18 +1,11 @@ %% -*- mode: erlang -*- {VSN, [ {<<".*">>, - %% load all plugins - %% NOTE: this depends on the fact that emqx_dashboard is always - %% the last application gets upgraded - [ {apply, {emqx_rule_engine, load_providers, []}} - , {restart_application, emqx_dashboard} - , {apply, {emqx_plugins, load, []}} + [ {restart_application, emqx_dashboard} ]} ], [ {<<".*">>, - [ {apply, {emqx_rule_engine, load_providers, []}} - , {restart_application, emqx_dashboard} - , {apply, {emqx_plugins, load, []}} + [ {restart_application, emqx_dashboard} ]} ] }. diff --git a/rebar.config.erl b/rebar.config.erl index ce8933570..492707867 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -128,30 +128,23 @@ prod_compile_opts() -> prod_overrides() -> [{add, [ {erl_opts, [deterministic]}]}]. -relup_deps(Profile) -> - {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "scripts/inject-deps.escript " ++ atom_to_list(Profile)}]}. - profiles() -> Vsn = get_vsn(), [ {'emqx', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, bin)} , {overrides, prod_overrides()} - , relup_deps('emqx') ]} , {'emqx-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, pkg)} , {overrides, prod_overrides()} - , relup_deps('emqx-pkg') ]} , {'emqx-edge', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, bin)} , {overrides, prod_overrides()} - , relup_deps('emqx-edge') ]} , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, pkg)} , {overrides, prod_overrides()} - , relup_deps('emqx-edge-pkg') ]} , {check, [ {erl_opts, common_compile_opts()} ]} diff --git a/scripts/inject-relup.escript b/scripts/inject-relup.escript new file mode 100755 index 000000000..e2d3de795 --- /dev/null +++ b/scripts/inject-relup.escript @@ -0,0 +1,139 @@ +#!/usr/bin/env escript + +%% This script injects implicit relup instructions for emqx applications. + +-mode(compile). + +-define(ERROR(FORMAT, ARGS), io:format(standard_error, "[inject-relup] " ++ FORMAT ++ "~n", ARGS)). +-define(INFO(FORMAT, ARGS), io:format(user, "[inject-relup] " ++ FORMAT ++ "~n", ARGS)). + +usage() -> + "Usage: " ++ escript:script_name() ++ " ". + +main([RelupFile]) -> + case filelib:is_regular(RelupFile) of + true -> + ok = inject_relup_file(RelupFile); + false -> + ?ERROR("not a valid file: ~p", [RelupFile]), + erlang:halt(1) + end; +main(_Args) -> + ?ERROR("~s", [usage()]), + erlang:halt(1). + +inject_relup_file(File) -> + case file:script(File) of + {ok, {CurrRelVsn, UpVsnRUs, DnVsnRUs}} -> + ?INFO("injecting instructions to: ~p", [File]), + UpdatedContent = {CurrRelVsn, + inject_relup_instrs(up, UpVsnRUs), + inject_relup_instrs(down, DnVsnRUs)}, + file:write_file(File, term_to_text(UpdatedContent)); + {ok, _BadFormat} -> + ?ERROR("bad formatted relup file: ~p", [File]), + error({bad_relup_format, File}); + {error, enoent} -> + ?INFO("relup file not found: ~p", [File]), + ok; + {error, Reason} -> + ?ERROR("read relup file ~p failed: ~p", [File, Reason]), + error({read_relup_error, Reason}) + end. + +inject_relup_instrs(Type, RUs) -> + lists:map(fun({Vsn, Desc, Instrs}) -> + {Vsn, Desc, append_emqx_relup_instrs(Type, Vsn, Instrs)} + end, RUs). + +append_emqx_relup_instrs(up, FromRelVsn, Instrs0) -> + {{UpExtra, _}, Instrs1} = filter_and_check_instrs(up, Instrs0), + Instrs1 ++ + [ {load, {emqx_app, brutal_purge, soft_purge}} + , {load, {emqx_relup, brutal_purge, soft_purge}} + , {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, UpExtra]}} + ]; + +append_emqx_relup_instrs(down, ToRelVsn, Instrs0) -> + {{_, DnExtra}, Instrs1} = filter_and_check_instrs(down, Instrs0), + %% NOTE: When downgrading, we apply emqx_relup:post_release_downgrade/2 before reloading + %% or removing the emqx_relup module. + Instrs2 = Instrs1 ++ + [ {load, {emqx_app, brutal_purge, soft_purge}} + , {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, DnExtra]}} + ], + %% emqx_relup does not exist before release "4.4.2" + LoadInsts = + case ToRelVsn of + ToRelVsn when ToRelVsn =:= "4.4.1"; ToRelVsn =:= "4.4.0" -> + [ {remove, {emqx_relup, brutal_purge, brutal_purge}} + , {purge, [emqx_relup]} + ]; + _ -> + [{load, {emqx_relup, brutal_purge, soft_purge}}] + end, + Instrs2 ++ LoadInsts. + +filter_and_check_instrs(Type, Instrs) -> + case filter_fetch_emqx_mods_and_extra(Instrs) of + {_, DnExtra, _, _} when Type =:= up, DnExtra =/= undefined -> + ?ERROR("got '{apply,{emqx_relup,post_release_downgrade,[_,Extra]}}'" + " from the upgrade instruction list, should be 'post_release_upgrade'", []), + error({instruction_not_found, load_object_code}); + {UpExtra, _, _, _} when Type =:= down, UpExtra =/= undefined -> + ?ERROR("got '{apply,{emqx_relup,post_release_upgrade,[_,Extra]}}'" + " from the downgrade instruction list, should be 'post_release_downgrade'", []), + error({instruction_not_found, load_object_code}); + {_, _, [], _} -> + ?ERROR("cannot find any 'load_object_code' instructions for app emqx", []), + error({instruction_not_found, load_object_code}); + {UpExtra, DnExtra, EmqxMods, RemainInstrs} -> + assert_mandatory_modules(Type, EmqxMods), + {{UpExtra, DnExtra}, RemainInstrs} + end. + +filter_fetch_emqx_mods_and_extra(Instrs) -> + lists:foldl(fun do_filter_and_get/2, {undefined, undefined, [], []}, Instrs). + +%% collect modules for emqx app +do_filter_and_get({load_object_code, {emqx, _AppVsn, Mods}} = Instr, + {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra, EmqxMods ++ Mods, RemainInstrs ++ [Instr]}; +%% remove 'load' instrs for emqx_relup and emqx_app +do_filter_and_get({load, {Mod, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) + when Mod =:= emqx_relup; Mod =:= emqx_app -> + {UpExtra, DnExtra, EmqxMods, RemainInstrs}; +%% remove 'remove' and 'purge' instrs for emqx_relup +do_filter_and_get({remove, {emqx_relup, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra, EmqxMods, RemainInstrs}; +do_filter_and_get({purge, [emqx_relup]}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra, EmqxMods, RemainInstrs}; +%% remove 'apply' instrs for upgrade, and collect the 'Extra' parameter +do_filter_and_get({apply, {emqx_relup, post_release_upgrade, [_, UpExtra0]}}, + {_, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra0, DnExtra, EmqxMods, RemainInstrs}; +%% remove 'apply' instrs for downgrade, and collect the 'Extra' parameter +do_filter_and_get({apply, {emqx_relup, post_release_downgrade, [_, DnExtra0]}}, + {UpExtra, _, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra0, EmqxMods, RemainInstrs}; +%% keep all other instrs unchanged +do_filter_and_get(Instr, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra, EmqxMods, RemainInstrs ++ [Instr]}. + + +assert_mandatory_modules(up, Mods) -> + assert(lists:member(emqx_relup, Mods) andalso lists:member(emqx_app, Mods), + "cannot find any 'load_object_code' instructions for emqx_app and emqx_rel: ~p", [Mods]); + +assert_mandatory_modules(down, Mods) -> + assert(lists:member(emqx_app, Mods), + "cannot find any 'load_object_code' instructions for emqx_app", []). + +assert(true, _, _) -> + ok; +assert(false, Msg, Args) -> + ?ERROR(Msg, Args), + error(assert_failed). + +term_to_text(Term) -> + io_lib:format("~p.", [Term]). diff --git a/scripts/pkg-full-vsn.sh b/scripts/pkg-full-vsn.sh index de32e11ef..e118643c9 100755 --- a/scripts/pkg-full-vsn.sh +++ b/scripts/pkg-full-vsn.sh @@ -27,7 +27,7 @@ esac cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" -SYSTEM="$(./scripts/get-distro.sh)" +SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" UNAME="$(uname -m)" case "$UNAME" in diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 2b2f7e92f..901460a24 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -28,6 +28,7 @@ case $PROFILE in esac SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" +OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" ARCH="${ARCH:-$(uname -m)}" case "$ARCH" in @@ -65,10 +66,11 @@ pushd _upgrade_base for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do filename="$PROFILE-${tag#[e|v]}-otp$OTP_VSN-$SYSTEM-$ARCH.zip" url="https://www.emqx.com/downloads/$DIR/${tag#[e|v]}/$filename" - echo "downloading base package from ${url} ..." if [ ! -f "$filename" ] && curl -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then + echo "downloading base package from ${url} ..." curl -L -o "${filename}" "${url}" if [ "$SYSTEM" != "centos6" ]; then + echo "downloading sha256 sum from ${url}.sha256 ..." curl -L -o "${filename}.sha256" "${url}.sha256" SUMSTR=$(cat "${filename}.sha256") echo "got sha265sum: ${SUMSTR}" diff --git a/src/emqx.appup.src b/src/emqx.appup.src index b9d4f5a16..53b4a9ff7 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,9 @@ %% -*- mode: erlang -*- {VSN, [{"4.4.1", - [{load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {add_module,emqx_relup}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -10,7 +12,8 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + [{add_module,emqx_relup}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, @@ -31,7 +34,9 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.1", - [{load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {delete_module,emqx_relup}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -40,7 +45,8 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_relup}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_relup.erl b/src/emqx_relup.erl new file mode 100644 index 000000000..fcc90d088 --- /dev/null +++ b/src/emqx_relup.erl @@ -0,0 +1,56 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_relup). + +%% NOTE: DO NOT remove this `-include`. +%% We use this to force this module to be upgraded every release. +-include("emqx_release.hrl"). + +-export([ post_release_upgrade/2 + , post_release_downgrade/2 + ]). + +-define(INFO(FORMAT), io:format("[emqx_relup] " ++ FORMAT ++ "~n")). +-define(INFO(FORMAT, ARGS), io:format("[emqx_relup] " ++ FORMAT ++ "~n", ARGS)). + +%% What to do after upgraded from an old release vsn. +post_release_upgrade(FromRelVsn, _) -> + {_, CurrRelVsn} = ?EMQX_RELEASE, + ?INFO("emqx has been upgraded to from ~s to ~s!", [FromRelVsn, CurrRelVsn]), + reload_components(). + +%% What to do after downgraded to an old release vsn. +post_release_downgrade(ToRelVsn, _) -> + {_, CurrRelVsn} = ?EMQX_RELEASE, + ?INFO("emqx has been downgraded to from ~s to ~s!", [CurrRelVsn, ToRelVsn]), + reload_components(). + +-ifdef(EMQX_ENTERPRISE). +reload_components() -> + ?INFO("reloading resource providers ..."), + emqx_rule_engine:load_providers(), + ?INFO("reloading module providers ..."), + emqx_modules:load_providers(), + ?INFO("loading plugins ..."), + emqx_plugins:load(). +-else. +reload_components() -> + ?INFO("reloading resource providers ..."), + emqx_rule_engine:load_providers(), + ?INFO("loading plugins ..."), + emqx_plugins:load(). +-endif.