Merge pull request #7147 from emqx/inject_relup_instructions_v4.4

Inject relup instructions v4.4

build

@@ -18,7 +18,7 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
 PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
 export PKG_VSN
 
-SYSTEM="$(./scripts/get-distro.sh)"
+SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}"
 
 ##
 ## Support RPM and Debian based linux systems
@@ -136,6 +136,9 @@ make_zip() {
     local tarball="${relpath}/${tarname}"
     local target_zip="${pkgpath}/${pkgname}"
     tar zxf "${tarball}" -C "${tard}/emqx"
+    if ! [[ $SYSTEM == windows* ]]; then
+        ./scripts/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup"
+    fi
     cp_dyn_libs "${tard}/emqx"
     pushd "${tard}" >/dev/null
     case "$SYSTEM" in

lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src

@@ -1,18 +1,11 @@
 %% -*- mode: erlang -*-
 {VSN,
  [ {<<".*">>,
-    %% load all plugins
+    [ {restart_application, emqx_dashboard}
-    %% NOTE: this depends on the fact that emqx_dashboard is always
-    %% the last application gets upgraded
-    [ {apply, {emqx_rule_engine, load_providers, []}}
-    , {restart_application, emqx_dashboard}
-    , {apply, {emqx_plugins, load, []}}
     ]}
  ],
  [ {<<".*">>,
-    [ {apply, {emqx_rule_engine, load_providers, []}}
+    [ {restart_application, emqx_dashboard}
-    , {restart_application, emqx_dashboard}
-    , {apply, {emqx_plugins, load, []}}
     ]}
  ]
 }.

rebar.config.erl

@@ -128,30 +128,23 @@ prod_compile_opts() ->
 prod_overrides() ->
     [{add, [ {erl_opts, [deterministic]}]}].
 
-relup_deps(Profile) ->
-    {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "scripts/inject-deps.escript " ++ atom_to_list(Profile)}]}.
-
 profiles() ->
     Vsn = get_vsn(),
     [ {'emqx',          [ {erl_opts, prod_compile_opts()}
                         , {relx, relx(Vsn, cloud, bin)}
                         , {overrides, prod_overrides()}
-                        , relup_deps('emqx')
                         ]}
     , {'emqx-pkg',      [ {erl_opts, prod_compile_opts()}
                         , {relx, relx(Vsn, cloud, pkg)}
                         , {overrides, prod_overrides()}
-                        , relup_deps('emqx-pkg')
                         ]}
     , {'emqx-edge',     [ {erl_opts, prod_compile_opts()}
                         , {relx, relx(Vsn, edge, bin)}
                         , {overrides, prod_overrides()}
-                        , relup_deps('emqx-edge')
                         ]}
     , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()}
                         , {relx, relx(Vsn, edge, pkg)}
                         , {overrides, prod_overrides()}
-                        , relup_deps('emqx-edge-pkg')
                         ]}
     , {check,           [ {erl_opts, common_compile_opts()}
                         ]}

scripts/inject-relup.escript

@@ -0,0 +1,139 @@
+#!/usr/bin/env escript
+
+%% This script injects implicit relup instructions for emqx applications.
+
+-mode(compile).
+
+-define(ERROR(FORMAT, ARGS), io:format(standard_error, "[inject-relup] " ++ FORMAT ++ "~n", ARGS)).
+-define(INFO(FORMAT, ARGS), io:format(user, "[inject-relup] " ++ FORMAT ++ "~n", ARGS)).
+
+usage() ->
+    "Usage: " ++ escript:script_name() ++ " <path-to-relup-file>".
+
+main([RelupFile]) ->
+    case filelib:is_regular(RelupFile) of
+        true ->
+            ok = inject_relup_file(RelupFile);
+        false ->
+            ?ERROR("not a valid file: ~p", [RelupFile]),
+            erlang:halt(1)
+    end;
+main(_Args) ->
+    ?ERROR("~s", [usage()]),
+    erlang:halt(1).
+
+inject_relup_file(File) ->
+    case file:script(File) of
+        {ok, {CurrRelVsn, UpVsnRUs, DnVsnRUs}} ->
+            ?INFO("injecting instructions to: ~p", [File]),
+            UpdatedContent = {CurrRelVsn,
+                inject_relup_instrs(up, UpVsnRUs),
+                inject_relup_instrs(down, DnVsnRUs)},
+            file:write_file(File, term_to_text(UpdatedContent));
+        {ok, _BadFormat} ->
+            ?ERROR("bad formatted relup file: ~p", [File]),
+            error({bad_relup_format, File});
+        {error, enoent} ->
+            ?INFO("relup file not found: ~p", [File]),
+            ok;
+        {error, Reason} ->
+            ?ERROR("read relup file ~p failed: ~p", [File, Reason]),
+            error({read_relup_error, Reason})
+    end.
+
+inject_relup_instrs(Type, RUs) ->
+    lists:map(fun({Vsn, Desc, Instrs}) ->
+        {Vsn, Desc, append_emqx_relup_instrs(Type, Vsn, Instrs)}
+    end, RUs).
+
+append_emqx_relup_instrs(up, FromRelVsn, Instrs0) ->
+     {{UpExtra, _}, Instrs1} = filter_and_check_instrs(up, Instrs0),
+     Instrs1 ++
+        [ {load, {emqx_app, brutal_purge, soft_purge}}
+        , {load, {emqx_relup, brutal_purge, soft_purge}}
+        , {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, UpExtra]}}
+        ];
+
+append_emqx_relup_instrs(down, ToRelVsn, Instrs0) ->
+    {{_, DnExtra}, Instrs1} = filter_and_check_instrs(down, Instrs0),
+    %% NOTE: When downgrading, we apply emqx_relup:post_release_downgrade/2 before reloading
+    %%  or removing the emqx_relup module.
+    Instrs2 = Instrs1 ++
+        [ {load, {emqx_app, brutal_purge, soft_purge}}
+        , {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, DnExtra]}}
+        ],
+    %% emqx_relup does not exist before release "4.4.2"
+    LoadInsts =
+        case ToRelVsn of
+            ToRelVsn when ToRelVsn =:= "4.4.1"; ToRelVsn =:= "4.4.0" ->
+                [ {remove, {emqx_relup, brutal_purge, brutal_purge}}
+                , {purge, [emqx_relup]}
+                ];
+            _ ->
+                [{load, {emqx_relup, brutal_purge, soft_purge}}]
+        end,
+    Instrs2 ++ LoadInsts.
+
+filter_and_check_instrs(Type, Instrs) ->
+    case filter_fetch_emqx_mods_and_extra(Instrs) of
+        {_, DnExtra, _, _} when Type =:= up, DnExtra =/= undefined ->
+            ?ERROR("got '{apply,{emqx_relup,post_release_downgrade,[_,Extra]}}'"
+                   " from the upgrade instruction list, should be 'post_release_upgrade'", []),
+            error({instruction_not_found, load_object_code});
+        {UpExtra, _, _, _} when Type =:= down, UpExtra =/= undefined ->
+            ?ERROR("got '{apply,{emqx_relup,post_release_upgrade,[_,Extra]}}'"
+                   " from the downgrade instruction list, should be 'post_release_downgrade'", []),
+            error({instruction_not_found, load_object_code});
+        {_, _, [], _} ->
+            ?ERROR("cannot find any 'load_object_code' instructions for app emqx", []),
+            error({instruction_not_found, load_object_code});
+        {UpExtra, DnExtra, EmqxMods, RemainInstrs} ->
+            assert_mandatory_modules(Type, EmqxMods),
+            {{UpExtra, DnExtra}, RemainInstrs}
+    end.
+
+filter_fetch_emqx_mods_and_extra(Instrs) ->
+    lists:foldl(fun do_filter_and_get/2, {undefined, undefined, [], []}, Instrs).
+
+%% collect modules for emqx app
+do_filter_and_get({load_object_code, {emqx, _AppVsn, Mods}} = Instr,
+        {UpExtra, DnExtra, EmqxMods, RemainInstrs}) ->
+    {UpExtra, DnExtra, EmqxMods ++ Mods, RemainInstrs ++ [Instr]};
+%% remove 'load' instrs for emqx_relup and emqx_app
+do_filter_and_get({load, {Mod, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs})
+        when Mod =:= emqx_relup; Mod =:= emqx_app ->
+    {UpExtra, DnExtra, EmqxMods, RemainInstrs};
+%% remove 'remove' and 'purge' instrs for emqx_relup
+do_filter_and_get({remove, {emqx_relup, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) ->
+    {UpExtra, DnExtra, EmqxMods, RemainInstrs};
+do_filter_and_get({purge, [emqx_relup]}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) ->
+    {UpExtra, DnExtra, EmqxMods, RemainInstrs};
+%% remove 'apply' instrs for upgrade, and collect the 'Extra' parameter
+do_filter_and_get({apply, {emqx_relup, post_release_upgrade, [_, UpExtra0]}},
+        {_, DnExtra, EmqxMods, RemainInstrs}) ->
+    {UpExtra0, DnExtra, EmqxMods, RemainInstrs};
+%% remove 'apply' instrs for downgrade, and collect the 'Extra' parameter
+do_filter_and_get({apply, {emqx_relup, post_release_downgrade, [_, DnExtra0]}},
+        {UpExtra, _, EmqxMods, RemainInstrs}) ->
+    {UpExtra, DnExtra0, EmqxMods, RemainInstrs};
+%% keep all other instrs unchanged
+do_filter_and_get(Instr, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) ->
+    {UpExtra, DnExtra, EmqxMods, RemainInstrs ++ [Instr]}.
+
+
+assert_mandatory_modules(up, Mods) ->
+    assert(lists:member(emqx_relup, Mods) andalso lists:member(emqx_app, Mods),
+        "cannot find any 'load_object_code' instructions for emqx_app and emqx_rel: ~p", [Mods]);
+
+assert_mandatory_modules(down, Mods) ->
+    assert(lists:member(emqx_app, Mods),
+        "cannot find any 'load_object_code' instructions for emqx_app", []).
+
+assert(true, _, _) ->
+    ok;
+assert(false, Msg, Args) ->
+    ?ERROR(Msg, Args),
+    error(assert_failed).
+
+term_to_text(Term) ->
+    io_lib:format("~p.", [Term]).

scripts/pkg-full-vsn.sh

@@ -27,7 +27,7 @@ esac
 cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
 
 OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}"
-SYSTEM="$(./scripts/get-distro.sh)"
+SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}"
 
 UNAME="$(uname -m)"
 case "$UNAME" in

scripts/relup-base-packages.sh

@@ -28,6 +28,7 @@ case $PROFILE in
 esac
 
 SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}"
+OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}"
 
 ARCH="${ARCH:-$(uname -m)}"
 case "$ARCH" in
@@ -65,10 +66,11 @@ pushd _upgrade_base
 for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do
     filename="$PROFILE-${tag#[e|v]}-otp$OTP_VSN-$SYSTEM-$ARCH.zip"
     url="https://www.emqx.com/downloads/$DIR/${tag#[e|v]}/$filename"
-    echo "downloading base package from ${url} ..."
     if [ ! -f "$filename" ] && curl -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then
+        echo "downloading base package from ${url} ..."
         curl -L -o "${filename}" "${url}"
         if [ "$SYSTEM" != "centos6" ]; then
+            echo "downloading sha256 sum from ${url}.sha256 ..."
             curl -L -o "${filename}.sha256" "${url}.sha256"
             SUMSTR=$(cat "${filename}.sha256")
             echo "got sha265sum: ${SUMSTR}"

src/emqx.appup.src

@@ -1,7 +1,9 @@
 %% -*- mode: erlang -*-
 {VSN,
   [{"4.4.1",
-    [{load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {add_module,emqx_relup},
+     {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
@@ -10,7 +12,8 @@
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.4.0",
-    [{load_module,emqx_pmon,brutal_purge,soft_purge,[]},
+    [{add_module,emqx_relup},
+     {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -31,7 +34,9 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{"4.4.1",
-    [{load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_relup},
+     {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
@@ -40,7 +45,8 @@
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.4.0",
-    [{load_module,emqx_pmon,brutal_purge,soft_purge,[]},
+    [{delete_module,emqx_relup},
+     {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},

src/emqx_relup.erl

@@ -0,0 +1,56 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_relup).
+
+%% NOTE: DO NOT remove this `-include`.
+%% We use this to force this module to be upgraded every release.
+-include("emqx_release.hrl").
+
+-export([ post_release_upgrade/2
+        , post_release_downgrade/2
+        ]).
+
+-define(INFO(FORMAT), io:format("[emqx_relup] " ++ FORMAT ++ "~n")).
+-define(INFO(FORMAT, ARGS), io:format("[emqx_relup] " ++ FORMAT ++ "~n", ARGS)).
+
+%% What to do after upgraded from an old release vsn.
+post_release_upgrade(FromRelVsn, _) ->
+    {_, CurrRelVsn} = ?EMQX_RELEASE,
+    ?INFO("emqx has been upgraded to from ~s to ~s!", [FromRelVsn, CurrRelVsn]),
+    reload_components().
+
+%% What to do after downgraded to an old release vsn.
+post_release_downgrade(ToRelVsn, _) ->
+    {_, CurrRelVsn} = ?EMQX_RELEASE,
+    ?INFO("emqx has been downgraded to from ~s to ~s!", [CurrRelVsn, ToRelVsn]),
+    reload_components().
+
+-ifdef(EMQX_ENTERPRISE).
+reload_components() ->
+    ?INFO("reloading resource providers ..."),
+    emqx_rule_engine:load_providers(),
+    ?INFO("reloading module providers ..."),
+    emqx_modules:load_providers(),
+    ?INFO("loading plugins ..."),
+    emqx_plugins:load().
+-else.
+reload_components() ->
+    ?INFO("reloading resource providers ..."),
+    emqx_rule_engine:load_providers(),
+    ?INFO("loading plugins ..."),
+    emqx_plugins:load().
+-endif.