Merge pull request #7172 from zmstone/chore-add-appup-update-script

chore: add appup update script

.github/workflows/apps_version_check.yaml

@@ -6,7 +6,22 @@ jobs:
   check_apps_version:
     runs-on: ubuntu-20.04
 
+    strategy:
+      matrix:
+        erl_otp:
+        - erl23.2.7.2-emqx-3
+        os:
+        - ubuntu20.04
+
+    container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }}
+
     steps:
       - uses: actions/checkout@v1
       - name: Check apps version
         run: ./scripts/apps-version-check.sh
+      - name: Check relup (ce)
+        if: endsWith(github.repository, 'emqx')
+        run: ./scripts/update-appup.sh emqx --check
+      - name: Check relup (ee)
+        if: endsWith(github.repository, 'enterprise')
+        run: ./scripts/update-appup.sh emqx-ee --check

apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src

@@ -13,12 +13,14 @@
      {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]},
    {<<"4.3.4">>,
-    [{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
    {<<"4.3.5">>,
-    [{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]},
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{<<"4.3.[0-3]">>,
     [{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
@@ -33,10 +35,12 @@
      {delete_module,emqx_acl_mnesia_migrator},
      {delete_module,emqx_acl_mnesia_db}]},
    {<<"4.3.4">>,
-    [{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
    {<<"4.3.5">>,
-    [{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]},
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}]}.

apps/emqx_rule_engine/src/emqx_rule_engine.app.src

@@ -1,6 +1,6 @@
 {application, emqx_rule_engine,
  [{description, "EMQ X Rule Engine"},
-  {vsn, "4.3.7"}, % strict semver, bump manually!
+  {vsn, "4.3.8"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_rule_engine_sup, emqx_rule_registry]},
   {applications, [kernel,stdlib,rulesql,getopt]},

apps/emqx_rule_engine/src/emqx_rule_engine.appup.src

@@ -1,6 +1,7 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.3.6",
+  [{"4.3.7",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.3.6",
     [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {update,emqx_rule_metrics,{advanced,["4.3.6"]}},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@@ -58,7 +59,8 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.6",
+  [{"4.3.7",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.3.6",
     [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {update,emqx_rule_metrics,{advanced,["4.3.6"]}},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},

apps/emqx_sn/rebar.config

@@ -2,7 +2,7 @@
 {plugins, [rebar3_proper]}.
 
 {deps,
- [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.7.4"}}},
+ [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.5"}}},
   {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}}
  ]}.
 

include/emqx_release.hrl

@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.3.12"}).
+-define(EMQX_RELEASE, {opensource, "4.3.13-rc.1"}).
 
 -else.
 

rebar.config

@@ -42,7 +42,7 @@
     , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}}
-    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.4"}}}
+    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.5"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.8"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
     , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}}

scripts/update-appup.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+## This script wrapps update_appup.escript,
+## it provides a more commonly used set of default args.
+
+## Arg1: EMQX PROFILE
+
+set -euo pipefail
+
+usage() {
+    echo "$0 PROFILE PREV_VERSION"
+}
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+
+PROFILE="${1:-}"
+case "$PROFILE" in
+    emqx-ee)
+        DIR='enterprise'
+        TAG_PREFIX='e'
+        ;;
+    emqx)
+        DIR='broker'
+        TAG_PREFIX='v'
+        ;;
+    emqx-edge)
+        DIR='edge'
+        TAG_PREFIX='v'
+        ;;
+    *)
+        echo "Unknown profile $PROFILE"
+        usage
+        exit 1
+        ;;
+esac
+
+PREV_VERSION="$(git describe --tag --match "${TAG_PREFIX}*" | grep -oE "${TAG_PREFIX}4\.[0-9]+\.[0-9]+")"
+PREV_VERSION="${PREV_VERSION#[e|v]}"
+
+shift 1
+ESCRIPT_ARGS="$*"
+
+SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}"
+ARCH="${ARCH:-$(uname -m)}"
+case "$ARCH" in
+    x86_64)
+        ARCH='amd64'
+        ;;
+    aarch64)
+        ARCH='arm64'
+        ;;
+    arm*)
+        ARCH=arm
+        ;;
+esac
+
+PACKAGE_NAME="${PROFILE}-${SYSTEM}-${PREV_VERSION}-${ARCH}.zip"
+DOWNLOAD_URL="https://www.emqx.com/downloads/${DIR}/v${PREV_VERSION}/${PACKAGE_NAME}"
+
+# shellcheck disable=SC2086
+./scripts/update_appup.escript --make-command "make ${PROFILE}-rel" --binary-rel-url "$DOWNLOAD_URL" $ESCRIPT_ARGS "$PREV_VERSION"

scripts/update_appup.escript

@@ -32,8 +32,8 @@ Options:
   --make-command    A command used to assemble the release
   --release-dir     Release directory
   --src-dirs        Directories where source code is found. Defaults to '{src,apps,lib-*}/**/'
-  --binary-rel-url  Binary release URL pattern. %TAG% variable is substituted with the release tag.
-                    E.g. \"https://github.com/emqx/emqx/releases/download/v%TAG%/emqx-centos7-%TAG%-amd64.zip\"
+  --binary-rel-url  Binary release URL pattern.
+                    E.g. https://www.emqx.com/downloads/broker/v4.3.12/emqx-centos7-4.3.12-amd64.zip
 ".
 
 -record(app,
@@ -171,9 +171,10 @@ download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) -
     BaseDir = "/tmp/emqx-baseline-bin/",
     Dir = filename:basename(Repo, ".git") ++ [$-|Tag],
     Filename = filename:join(BaseDir, Dir),
-    Script = "mkdir -p ${OUTFILE} &&
-              wget -c -O ${OUTFILE}.zip ${URL} &&
-              unzip -n -d ${OUTFILE} ${OUTFILE}.zip",
+    Script = "echo \"Download: ${OUTFILE}\" &&
+              mkdir -p ${OUTFILE} &&
+              curl -f -L -o ${OUTFILE}.zip ${URL} &&
+              unzip -q -n -d ${OUTFILE} ${OUTFILE}.zip",
     Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}],
     bash(Script, Env),
     {ok, Filename}.
@@ -208,8 +209,8 @@ find_appup_actions(App,
     {OldUpgrade0, OldDowngrade0} = find_old_appup_actions(App, PrevVersion),
     OldUpgrade = ensure_all_patch_versions(App, CurrVersion, OldUpgrade0),
     OldDowngrade = ensure_all_patch_versions(App, CurrVersion, OldDowngrade0),
-    Upgrade = merge_update_actions(App, diff_app(App, CurrAppIdx, PrevAppIdx), OldUpgrade),
-    Downgrade = merge_update_actions(App, diff_app(App, PrevAppIdx, CurrAppIdx), OldDowngrade),
+    Upgrade = merge_update_actions(App, diff_app(up, App, CurrAppIdx, PrevAppIdx), OldUpgrade),
+    Downgrade = merge_update_actions(App, diff_app(down, App, PrevAppIdx, CurrAppIdx), OldDowngrade),
     if OldUpgrade =:= Upgrade andalso OldDowngrade =:= Downgrade ->
             %% The appup file has been already updated:
             [];
@@ -521,7 +522,7 @@ index_app(AppFile) ->
               , modules       = Modules
               }}.
 
-diff_app(App,
+diff_app(UpOrDown, App,
          #app{version = NewVersion, modules = NewModules},
          #app{version = OldVersion, modules = OldModules}) ->
     {New, Changed} =
@@ -539,18 +540,31 @@ diff_app(App,
                  , NewModules
                  ),
     Deleted = maps:keys(maps:without(maps:keys(NewModules), OldModules)),
-    NChanges = length(New) + length(Changed) + length(Deleted),
-    if NewVersion =:= OldVersion andalso NChanges > 0 ->
-            set_invalid(),
-            log("ERROR: Application '~p' contains changes, but its version is not updated~n", [App]);
-       NewVersion > OldVersion ->
-            log("INFO: Application '~p' has been updated: ~p -> ~p~n", [App, OldVersion, NewVersion]),
+    Changes = lists:filter(fun({_T, L}) -> length(L) > 0 end,
+                           [{added, New}, {changed, Changed}, {deleted, Deleted}]),
+    case NewVersion =:= OldVersion of
+        true when Changes =:= [] ->
+            %% no change
             ok;
-       true ->
+        true ->
+            set_invalid(),
+            case UpOrDown =:= up of
+                true ->
+                    %% only log for the upgrade case because it would be the same result
+                    log("ERROR: Application '~p' contains changes, but its version is not updated. ~s",
+                        [App, format_changes(Changes)]);
+                false ->
+                    ok
+            end;
+        false ->
+            log("INFO: Application '~p' has been updated: ~p --[~p]--> ~p~n", [App, OldVersion, UpOrDown, NewVersion]),
             ok
     end,
     {New, Changed, Deleted}.
 
+format_changes(Changes) ->
+    lists:map(fun({Tag, List}) -> io_lib:format("~p: ~p~n", [Tag, List]) end, Changes).
+
 -spec hashsums(file:filename()) -> #{module() => binary()}.
 hashsums(EbinDir) ->
     maps:from_list(lists:map(
@@ -607,13 +621,20 @@ locate(ebin_current, App, Suffix) ->
 locate(src, App, Suffix) ->
     AppStr = atom_to_list(App),
     SrcDirs = getopt(src_dirs),
-    case filelib:wildcard(SrcDirs ++ AppStr ++ Suffix) of
+    case find_app(SrcDirs ++ AppStr ++ Suffix) of
         [File] ->
             {ok, File};
         [] ->
-            undefined
+            undefined;
+        Files ->
+            error({more_than_one_app_found, Files})
     end.
 
+find_app(Pattern) ->
+    %% exclude _build dir inside apps
+    lists:filter(fun(S) -> string:find(S, "/_build/") =:= nomatch end,
+                 filelib:wildcard(Pattern)).
+
 bash(Script) ->
     bash(Script, []).
 

src/emqx.appup.src

@@ -1,7 +1,6 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [
-    {"4.3.13",
+  [{"4.3.13",
     [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys,brutal_purge,soft_purge,[]},
@@ -10,6 +9,7 @@
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.12",
     [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
@@ -390,9 +390,9 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [
-   {"4.3.13",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+  [{"4.3.13",
+    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},