Merge pull request #8283 from zmstone/0620-change-community-to-opensource-also-fix-relup

0620 change community to opensource also fix relup

apps/emqx/include/emqx_release.hrl

@@ -29,7 +29,7 @@
 %% hyphen.
 
 %% Community edition
--define(EMQX_RELEASE_CE, "5.0.0").
+-define(EMQX_RELEASE_CE, "5.0.1").
 
 %% Enterprise edition
 -define(EMQX_RELEASE_EE, "5.0.0-alpha.1").

apps/emqx/src/emqx.app.src

@@ -3,7 +3,7 @@
     {id, "emqx"},
     {description, "EMQX Core"},
     % strict semver, bump manually!
-    {vsn, "5.0.0"},
+    {vsn, "5.0.1"},
     {modules, []},
     {registered, []},
     {applications, [

apps/emqx/src/emqx.appup.src

@@ -0,0 +1,11 @@
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
+{VSN,
+  [{"5.0.0",[
+    {load_module,emqx_release,brutal_purge,soft_purge,[]},
+    {load_module,emqx_relup}]},
+   {<<".*">>,[]}],
+  [{"5.0.0",[
+    {load_module,emqx_release,brutal_purge,soft_purge,[]},
+    {load_module,emqx_relup}]},
+   {<<".*">>,[]}]}.

apps/emqx_dashboard/i18n/emqx_dashboard_api_i18n.conf

@@ -30,7 +30,7 @@ emqx_dashboard_api {
 
     license {
         desc {
-            en: """EMQX License. Community or enterprise"""
+            en: """EMQX License. opensource or enterprise"""
             zh: """EMQX 许可。开源版本 或者企业版"""
         }
     }

apps/emqx_dashboard/src/emqx_dashboard.app.src

@@ -2,7 +2,7 @@
 {application, emqx_dashboard, [
     {description, "EMQX Web Dashboard"},
     % strict semver, bump manually!
-    {vsn, "5.0.0"},
+    {vsn, "5.0.1"},
     {modules, []},
     {registered, [emqx_dashboard_sup]},
     {applications, [kernel, stdlib, mnesia, minirest, emqx]},

apps/emqx_dashboard/src/emqx_dashboard.appup.src

@@ -0,0 +1,7 @@
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
+{VSN,
+  [{"5.0.0",[{load_module,emqx_dashboard_api,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}],
+  [{"5.0.0",[{load_module,emqx_dashboard_api,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]}.

apps/emqx_dashboard/src/emqx_dashboard_api.erl

@@ -196,8 +196,8 @@ field(license) ->
     {license, [
         {edition,
             mk(
-                enum([community, enterprise]),
+                enum([opensource, enterprise]),
-                #{desc => ?DESC(license), example => community}
+                #{desc => ?DESC(license), example => opensource}
             )}
     ]};
 field(version) ->

apps/emqx_exhook/src/emqx_exhook.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_exhook, [
     {description, "EMQX Extension for Hook"},
-    {vsn, "5.0.0"},
+    {vsn, "5.0.1"},
     {modules, []},
     {registered, []},
     {mod, {emqx_exhook_app, []}},

apps/emqx_exhook/src/emqx_exhook.appup.src

@@ -1,8 +1,7 @@
 %% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-    [
+  [{"5.0.0",[{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}]},
-        {<<".*">>, []}
+   {<<".*">>,[]}],
-    ],
+  [{"5.0.0",[{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}]},
-    [
+   {<<".*">>,[]}]}.
-        {<<".*">>, []}
-    ]}.

apps/emqx_gateway/src/emqx_gateway.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_gateway, [
     {description, "The Gateway management application"},
-    {vsn, "0.1.0"},
+    {vsn, "0.1.1"},
     {registered, []},
     {mod, {emqx_gateway_app, []}},
     {applications, [kernel, stdlib, grpc, emqx, emqx_authn]},

apps/emqx_gateway/src/emqx_gateway.appup.src

@@ -0,0 +1,9 @@
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
+{VSN,
+  [{"0.1.0",[{load_module,emqx_exproto_pb,brutal_purge,soft_purge,[]}]},
+   {"0.1.0",[{load_module,emqx_exproto_pb,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}],
+  [{"0.1.0",[{load_module,emqx_exproto_pb,brutal_purge,soft_purge,[]}]},
+   {"0.1.0",[{load_module,emqx_exproto_pb,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]}.

apps/emqx_modules/src/emqx_modules.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_modules, [
     {description, "EMQX Modules"},
-    {vsn, "5.0.0"},
+    {vsn, "5.0.1"},
     {modules, []},
     {applications, [kernel, stdlib, emqx]},
     {mod, {emqx_modules_app, []}},

apps/emqx_modules/src/emqx_modules.appup.src

@@ -0,0 +1,11 @@
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
+{VSN,
+  [{"5.0.0",
+    [{load_module,emqx_telemetry_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_telemetry,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}],
+  [{"5.0.0",
+    [{load_module,emqx_telemetry_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_telemetry,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]}.

apps/emqx_modules/src/emqx_telemetry.erl

@@ -326,7 +326,7 @@ get_telemetry(State0 = #state{node_uuid = NodeUUID, cluster_uuid = ClusterUUID})
     } = get_rule_engine_and_bridge_info(),
     {State, [
         {emqx_version, bin(emqx_app:get_release())},
-        {license, [{edition, <<"community">>}]},
+        {license, [{edition, <<"opensource">>}]},
         {os_name, bin(get_value(os_name, OSInfo))},
         {os_version, bin(get_value(os_version, OSInfo))},
         {otp_version, bin(otp_version())},

apps/emqx_modules/src/emqx_telemetry_api.erl

@@ -106,7 +106,7 @@ fields(telemetry) ->
                 map(),
                 #{
                     desc => ?DESC(license),
-                    example => #{edition => <<"community">>}
+                    example => #{edition => <<"opensource">>}
                 }
             )},
         {os_name,

apps/emqx_retainer/src/emqx_retainer.app.src

@@ -2,7 +2,7 @@
 {application, emqx_retainer, [
     {description, "EMQX Retainer"},
     % strict semver, bump manually!
-    {vsn, "5.0.0"},
+    {vsn, "5.0.1"},
     {modules, []},
     {registered, [emqx_retainer_sup]},
     {applications, [kernel, stdlib, emqx]},

apps/emqx_retainer/src/emqx_retainer.appup.src

@@ -0,0 +1,7 @@
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
+{VSN,
+  [{"5.0.0",[{load_module,emqx_retainer_mnesia,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}],
+  [{"5.0.0",[{load_module,emqx_retainer_mnesia,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]}.

build

@@ -133,8 +133,15 @@ make_relup() {
         local tmp_dir
         tmp_dir="$(mktemp -d -t emqx.XXXXXXX)"
         $TAR -C "$tmp_dir" -zxf "$tgzfile"
-        cp -npr "$tmp_dir/releases"/* "${rel_dir}/releases/" || true
+        mkdir -p "${rel_dir}/releases/"
-        cp -npr "$tmp_dir/lib"/* "${rel_dir}/lib/" || true
+        cp -npr "$tmp_dir/releases"/* "${rel_dir}/releases/"
+        ## There is for some reason a copy of the '$PROFILE.rel' file to releases dir,
+        ## the content is duplicated to releases/5.0.0/$PROFILE.rel.
+        ## This file seems to be useless, but yet confusing as it does not change after upgrade/downgrade
+        ## Hence we force delete this file.
+        rm -f "${rel_dir}/releases/${PROFILE}.rel"
+        mkdir -p "${rel_dir}/lib/"
+        cp -npr "$tmp_dir/lib"/* "${rel_dir}/lib/"
         rm -rf "$tmp_dir"
         releases+=( "$base_vsn" )
     done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.tar.gz" -type f)

rebar.config.erl

@@ -207,7 +207,7 @@ profiles_dev() ->
 
 %% RelType: cloud (full size)
 %% PkgType: bin | pkg
-%% Edition: ce (community) | ee (enterprise)
+%% Edition: ce (opensource) | ee (enterprise)
 relx(Vsn, RelType, PkgType, Edition) ->
     [
         {include_src, false},

scripts/apps-version-check.sh

@@ -1,77 +1,54 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-## compare to the latest 5.0 release version tag:
+latest_release=$(git describe --abbrev=0 --tags --exclude '*rc*' --exclude '*alpha*' --exclude '*beta*')
-## but do not include alpha, beta and rc versions
-latest_release="$(git describe --abbrev=0 --tags --match '[v|e]5.0*' --exclude '*beta*' --exclude '*alpha*' --exclude '*rc*' || echo 'nomatch')"
-
-if [ "$latest_release" = 'nomatch' ]; then
-    echo "No base release found, skipping app vsn checks"
-    exit 0
-fi
-
-echo "Compare base: $latest_release"
 
 bad_app_count=0
 
-get_vsn() {
+no_comment_re='(^[^\s?%])'
-    commit="$1"
+## TODO: c source code comments re (in $app_path/c_src dirs)
-    app_src_file="$2"
+
-    if [ "$commit" = 'HEAD' ]; then
+parse_semver() {
-        if [ -f "$app_src_file" ]; then
+    echo "$1" | tr '.|-' ' '
-            grep vsn "$app_src_file" | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"' || true
-        fi
-    else
-        git show "$commit":"$app_src_file" 2>/dev/null | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"' || true
-    fi
 }
 
-check_apps() {
+while read -r app; do
-    while read -r app_path; do
+    if [ "$app" != "emqx" ]; then
-        app=$(basename "$app_path")
+        app_path="$app"
-        src_file="$app_path/src/$app.app.src"
+    else
-        old_app_version="$(get_vsn "$latest_release" "$src_file")"
+        app_path="."
-        ## TODO: delete it after new version is released with emqx app in apps dir
-        if [ "$app" = 'emqx' ] && [ "$old_app_version" = '' ]; then
-            old_app_version="$(get_vsn "$latest_release" 'src/emqx.app.src')"
     fi
-        now_app_version="$(get_vsn 'HEAD' "$src_file")"
+    src_file="$app_path/src/$(basename "$app").app.src"
-        ## TODO: delete it after new version is released with emqx app in apps dir
+    old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"')"
-        if [ "$app" = 'emqx' ] && [ "$now_app_version" = '' ]; then
+    now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"')
-            now_app_version="$(get_vsn 'HEAD' 'src/emqx.app.src')"
+    if [ "$old_app_version" = "$now_app_version" ]; then
-        fi
+        changed_lines="$(git diff "$latest_release"...HEAD --ignore-blank-lines -G "$no_comment_re" \
-        if [ -z "$now_app_version" ]; then
-            echo "failed_to_get_new_app_vsn for $app"
-            exit 1
-        fi
-        if [ -z "${old_app_version:-}" ]; then
-            echo "skipped checking new app ${app}"
-        elif [ "$old_app_version" = "$now_app_version" ]; then
-            lines="$(git diff --name-only "$latest_release"...HEAD \
                              -- "$app_path/src" \
+                             -- ":(exclude)"$app_path/src/*.appup.src"" \
                              -- "$app_path/priv" \
-                        -- "$app_path/c_src")"
+                             -- "$app_path/c_src" | wc -l ) "
-            if [ "$lines" != '' ]; then
+        if [ "$changed_lines" -gt 0 ]; then
-                echo "$src_file needs a vsn bump (old=$old_app_version)"
+            echo "$src_file needs a vsn bump"
-                echo "changed: $lines"
+            bad_app_count=$(( bad_app_count + 1))
+        fi
+    else
+        # shellcheck disable=SC2207
+        old_app_version_semver=($(parse_semver "$old_app_version"))
+        # shellcheck disable=SC2207
+        now_app_version_semver=($(parse_semver "$now_app_version"))
+        if  [ "${old_app_version_semver[0]}" = "${now_app_version_semver[0]}" ] && \
+            [ "${old_app_version_semver[1]}" = "${now_app_version_semver[1]}" ] && \
+            [ "$(( old_app_version_semver[2] + 1 ))" = "${now_app_version_semver[2]}" ]; then
+            true
+        else
+            echo "$src_file: non-strict semver version bump from $old_app_version to $now_app_version"
             bad_app_count=$(( bad_app_count + 1))
         fi
     fi
-    done < <(./scripts/find-apps.sh)
+done < <(./scripts/find-apps.sh)
 
-    if [ $bad_app_count -gt 0 ]; then
+if [ $bad_app_count -gt 0 ]; then
     exit 1
-    else
+else
     echo "apps version check successfully"
-    fi
+fi
-}
-
-_main() {
-    if echo "${latest_release}" |grep -oE '[0-9]+.[0-9]+.[0-9]+' > /dev/null 2>&1; then
-        check_apps
-    else
-        echo "skipped unstable tag: ${latest_release}"
-    fi
-}
-
-_main

scripts/relup-base-packages.sh

@@ -15,7 +15,7 @@ export PROFILE
 case $PROFILE in
     "emqx")
         DIR='broker'
-        EDITION='community'
+        EDITION='opensource'
         ;;
     "emqx-enterprise")
         DIR='enterprise'

scripts/update-appup.sh

@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+
+## This script wrapps update_appup.escript,
+## it provides a more commonly used set of default args.
+
+## Arg1: EMQX PROFILE
+
+set -euo pipefail
+set -x
+
+usage() {
+    echo "$0 PROFILE"
+}
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+
+PROFILE="${1:-}"
+GIT_REPO='emqx/emqx.git'
+case "$PROFILE" in
+    emqx-enterprise)
+        TAG_PREFIX='e'
+        ;;
+    emqx)
+        TAG_PREFIX='v'
+        ;;
+    *)
+        echo "Unknown profile $PROFILE"
+        usage
+        exit 1
+        ;;
+esac
+
+## possible tags:
+##  v4.3.11
+##  e4.3.11
+##  rel-v4.4.3
+##  rel-e4.4.3
+PREV_TAG="${PREV_TAG:-$(git describe --tag --abbrev=0 --match "[${TAG_PREFIX}|rel-]*" --exclude '*rc*' --exclude '*alpha*' --exclude '*beta*')}"
+
+shift 1
+# bash 3.2 treat empty array as unbound, so we can't use 'ESCRIPT_ARGS=()' here,
+# but must add an empty-string element to the array
+ESCRIPT_ARGS=( '' )
+while [ "$#" -gt 0 ]; do
+    case $1 in
+    -h|--help)
+        help
+        exit 0
+        ;;
+    --skip-build)
+        SKIP_BUILD='yes'
+        shift
+        ;;
+    --skip-build-base)
+        SKIP_BUILD_BASE='yes'
+        shift
+        ;;
+    --check)
+        # hijack the --check option
+        IS_CHECK='yes'
+        shift
+        ;;
+    *)
+        ESCRIPT_ARGS+=( "$1" )
+        shift
+        ;;
+  esac
+done
+
+if [ "$TAG_PREFIX" = 'v' ]; then
+    SRC_DIRS="{apps}"
+else
+    SRC_DIRS="{apps,lib-ee}"
+fi
+
+## make sure we build here in bash and always pass --skip-build to escript
+if [ "${SKIP_BUILD:-}" != 'yes' ]; then
+    make "${PROFILE}"
+fi
+
+PREV_DIR_BASE="/tmp/_w"
+mkdir -p "${PREV_DIR_BASE}"
+if [ ! -d "${PREV_DIR_BASE}/${PREV_TAG}" ]; then
+    cp -R . "${PREV_DIR_BASE}/${PREV_TAG}"
+    # always 'yes' in CI
+    NEW_COPY='yes'
+else
+    NEW_COPY='no'
+fi
+
+if [ "${SKIP_BUILD_BASE:-no}" = 'yes' ]; then
+    echo "not building relup base ${PREV_DIR_BASE}/${PREV_TAG}"
+else
+    pushd "${PREV_DIR_BASE}/${PREV_TAG}"
+    if [ "$NEW_COPY" = 'no' ]; then
+        REMOTE="$(git remote -v | grep "${GIT_REPO}" | head -1 | awk '{print $1}')"
+        git fetch "$REMOTE"
+    fi
+    git reset --hard
+    git clean -ffdx
+    git checkout "${PREV_TAG}"
+    make "$PROFILE"
+    popd
+fi
+
+PREV_REL_DIR="${PREV_DIR_BASE}/${PREV_TAG}/_build/${PROFILE}/lib"
+
+# bash 3.2 does not allow empty array, so we had to add an empty string in the ESCRIPT_ARGS array,
+# this in turn makes quoting "${ESCRIPT_ARGS[@]}" problematic, hence disable SC2068 check here
+# shellcheck disable=SC2068
+./scripts/update_appup.escript \
+    --src-dirs "${SRC_DIRS}/**" \
+    --release-dir "_build/${PROFILE}/lib" \
+    --prev-release-dir "${PREV_REL_DIR}" \
+    --skip-build \
+    ${ESCRIPT_ARGS[@]} "$PREV_TAG"
+
+if [ "${IS_CHECK:-}" = 'yes' ]; then
+    diffs="$(git diff --name-only | grep -E '\.appup\.src' || true)"
+    if [ "$diffs" != '' ]; then
+        git --no-pager diff
+        echo "$0 ---check produced git diff"
+        exit 1
+    fi
+fi

scripts/update_appup.escript

@@ -26,14 +26,13 @@ Usage:
 Options:
 
   --check            Don't update the appfile, just check that they are complete
-  --repo            Upstream git repo URL
+  --repo             Upsteam git repo URL
   --remote           Get upstream repo URL from the specified git remote
   --skip-build       Don't rebuild the releases. May produce wrong results
   --make-command     A command used to assemble the release
+  --prev-release-dir Previous version's release dir (if already built/extracted)
   --release-dir      Release directory
   --src-dirs         Directories where source code is found. Defaults to '{src,apps,lib-*}/**/'
-  --binary-rel-url  Binary release URL pattern. %VSN% variable is substituted with the version in release tag.
-                    E.g. \"https://github.com/emqx/emqx/releases/download/v%VSN%/emqx-%VSN%-otp-24.2.1-1-el7-amd64.tar.gz\"
 ".
 
 -record(app,
@@ -48,7 +47,7 @@ default_options() ->
      , check          => false
      , prev_tag       => undefined
      , src_dirs       => "{src,apps,lib-*}/**/"
-     , binary_rel_url => undefined
+     , prev_beams_dir => undefined
      }.
 
 %% App-specific actions that should be added unconditionally to any update/downgrade:
@@ -56,7 +55,8 @@ app_specific_actions(_) ->
     [].
 
 ignored_apps() ->
-    [emqx_dashboard, emqx_management] ++ otp_standard_apps().
+    [gpb %% only a build tool
+    ] ++ otp_standard_apps().
 
 main(Args) ->
     #{prev_tag := Baseline} = Options = parse_args(Args, default_options()),
@@ -68,7 +68,7 @@ parse_args([PrevTag = [A|_]], State) when A =/= $- ->
 parse_args(["--check"|Rest], State) ->
     parse_args(Rest, State#{check => true});
 parse_args(["--skip-build"|Rest], State) ->
-    parse_args(Rest, State#{make_command => "true"});
+    parse_args(Rest, State#{make_command => undefined});
 parse_args(["--repo", Repo|Rest], State) ->
     parse_args(Rest, State#{clone_url => Repo});
 parse_args(["--remote", Remote|Rest], State) ->
@@ -77,15 +77,16 @@ parse_args(["--make-command", Command|Rest], State) ->
     parse_args(Rest, State#{make_command => Command});
 parse_args(["--release-dir", Dir|Rest], State) ->
     parse_args(Rest, State#{beams_dir => Dir});
+parse_args(["--prev-release-dir", Dir|Rest], State) ->
+    parse_args(Rest, State#{prev_beams_dir => Dir});
 parse_args(["--src-dirs", Pattern|Rest], State) ->
     parse_args(Rest, State#{src_dirs => Pattern});
-parse_args(["--binary-rel-url", URL|Rest], State) ->
-    parse_args(Rest, State#{binary_rel_url => {ok, URL}});
 parse_args(_, _) ->
     fail(usage()).
 
 main(Options, Baseline) ->
     {CurrRelDir, PrevRelDir} = prepare(Baseline, Options),
+    putopt(prev_beams_dir, PrevRelDir),
     log("~n===================================~n"
         "Processing changes..."
         "~n===================================~n"),
@@ -93,38 +94,9 @@ main(Options, Baseline) ->
     PrevAppsIdx = index_apps(PrevRelDir),
     %% log("Curr: ~p~nPrev: ~p~n", [CurrAppsIdx, PrevAppsIdx]),
     AppupChanges = find_appup_actions(CurrAppsIdx, PrevAppsIdx),
-    case getopt(check) of
+    ok = update_appups(AppupChanges),
-        true ->
+    ok = check_appup_files(),
-            case AppupChanges of
+    ok = warn_and_exit(is_valid()).
-                [] ->
-                    ok;
-                _ ->
-                    Diffs =
-                        lists:filtermap(
-                          fun({App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}) ->
-                                  case parse_appup_diffs(Upgrade, OldUpgrade,
-                                                         Downgrade, OldDowngrade) of
-                                      ok ->
-                                          false;
-                                      {diffs, Diffs} ->
-                                          {true, {App, Diffs}}
-                                  end
-                          end,
-                          AppupChanges),
-                    case Diffs =:= [] of
-                        true ->
-                            ok;
-                        false ->
-                            set_invalid(),
-                            log("ERROR: The appup files are incomplete. Missing changes:~n   ~p",
-                                [Diffs])
-                    end
-            end;
-        false ->
-            update_appups(AppupChanges)
-    end,
-    check_appup_files(),
-    warn_and_exit(is_valid()).
 
 warn_and_exit(true) ->
     log("
@@ -136,47 +108,34 @@ warn_and_exit(false) ->
     log("~nERROR: Incomplete appups found. Please inspect the output for more details.~n"),
     halt(1).
 
-prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir, binary_rel_url := BinRel}) ->
+prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir}) ->
     log("~n===================================~n"
         "Baseline: ~s"
         "~n===================================~n", [Baseline]),
     log("Building the current version...~n"),
-    bash(MakeCommand),
+    ok = bash(MakeCommand),
-    log("Downloading and building the previous release...~n"),
     PrevRelDir =
-        case BinRel of
+        case maps:get(prev_beams_dir, Options, undefined) of
             undefined ->
+                log("Building the previous release...~n"),
                 {ok, PrevRootDir} = build_prev_release(Baseline, Options),
                 filename:join(PrevRootDir, BeamDir);
-            {ok, _URL} ->
+            Dir ->
-                {ok, PrevRootDir} = download_prev_release(Baseline, Options),
+                %% already built
-                PrevRootDir
+                Dir
         end,
     {BeamDir, PrevRelDir}.
 
 build_prev_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) ->
-    BaseDir = "/tmp/emqx-baseline/",
+    BaseDir = "/tmp/emqx-appup-base/",
     Dir = filename:basename(Repo, ".git") ++ [$-|Baseline],
-    %% TODO: shallow clone
     Script = "mkdir -p ${BASEDIR} &&
               cd ${BASEDIR} &&
-              { [ -d ${DIR} ] || git clone --branch ${TAG} ${REPO} ${DIR}; } &&
+              { [ -d ${DIR} ] || git clone --depth 1 --branch ${TAG} ${REPO} ${DIR}; } &&
               cd ${DIR} &&" ++ MakeCommand,
     Env = [{"REPO", Repo}, {"TAG", Baseline}, {"BASEDIR", BaseDir}, {"DIR", Dir}],
-    bash(Script, Env),
+    ok = bash(Script, Env),
-    {ok, filename:join(BaseDir, Dir)}.
+    {ok, filename:join([BaseDir, Dir, "_build/*/lib"])}.
-
-download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) ->
-    URL = string:replace(URL0, "%TAG%", Tag, all),
-    BaseDir = "/tmp/emqx-baseline-bin/",
-    Dir = filename:basename(Repo, ".git") ++ [$-|Tag],
-    Filename = filename:join(BaseDir, Dir),
-    Script = "mkdir -p ${OUTFILE} &&
-              wget -c -O ${OUTFILE}.tar.gz ${URL} &&
-              tar -zxf ${OUTFILE}.tar.gz -C ${OUTFILE}",
-    Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}],
-    bash(Script, Env),
-    {ok, Filename}.
 
 find_upstream_repo(Remote) ->
     string:trim(os:cmd("git remote get-url " ++ Remote)).
@@ -205,16 +164,16 @@ find_appup_actions(_App, AppIdx, AppIdx) ->
 find_appup_actions(App,
                    CurrAppIdx = #app{version = CurrVersion},
                    PrevAppIdx = #app{version = PrevVersion}) ->
-    {OldUpgrade0, OldDowngrade0} = find_old_appup_actions(App, PrevVersion),
+    {OldUpgrade0, OldDowngrade0} = find_base_appup_actions(App, PrevVersion),
     OldUpgrade = ensure_all_patch_versions(App, CurrVersion, OldUpgrade0),
     OldDowngrade = ensure_all_patch_versions(App, CurrVersion, OldDowngrade0),
-    Upgrade = merge_update_actions(App, diff_app(App, CurrAppIdx, PrevAppIdx), OldUpgrade),
+    UpDiff = diff_app(up, App, CurrAppIdx, PrevAppIdx),
-    Downgrade = merge_update_actions(App, diff_app(App, PrevAppIdx, CurrAppIdx), OldDowngrade),
+    DownDiff = diff_app(down, App, PrevAppIdx, CurrAppIdx),
-    if OldUpgrade =:= Upgrade andalso OldDowngrade =:= Downgrade ->
+    Upgrade = merge_update_actions(App, UpDiff, OldUpgrade, PrevVersion),
-            %% The appup file has been already updated:
+    Downgrade = merge_update_actions(App, DownDiff, OldDowngrade, PrevVersion),
-            [];
+    case OldUpgrade =:= Upgrade andalso OldDowngrade =:= Downgrade of
-       true ->
+        true -> [];
-            [{App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}]
+        false -> [{App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}]
     end.
 
 %% To avoid missing one patch version when upgrading, we try to
@@ -265,7 +224,7 @@ diff_appup_instructions(ComputedChanges, PresentChanges) ->
       [],
       ComputedChanges).
 
-%% For external dependencies, checks if any missing diffs are present
+%% checks if any missing diffs are present
 %% and groups them by `up' and `down' types.
 parse_appup_diffs(Upgrade, OldUpgrade, Downgrade, OldDowngrade) ->
     DiffUp = diff_appup_instructions(Upgrade, OldUpgrade),
@@ -275,61 +234,91 @@ parse_appup_diffs(Upgrade, OldUpgrade, Downgrade, OldDowngrade) ->
             %% no diff for external dependency; ignore
             ok;
         _ ->
-            set_invalid(),
             Diffs = #{ up => DiffUp
                      , down => DiffDown
                      },
             {diffs, Diffs}
     end.
 
+%% TODO: handle regexes
+%% Since the first argument may be a regex itself, we would need to
+%% check if it is "contained" within other regexes inside list of
+%% versions in the second argument.
 find_matching_version(VsnOrRegex, PresentChanges) ->
     proplists:get_value(VsnOrRegex, PresentChanges).
 
-find_old_appup_actions(App, PrevVersion) ->
+find_base_appup_actions(App, PrevVersion) ->
-    {Upgrade0, Downgrade0} =
+    {Upgrade, Downgrade} =
-        case locate(ebin_current, App, ".appup") of
+        case locate_appup(App) of
-            {ok, AppupFile} ->
-                log("Found the previous appup file: ~s~n", [AppupFile]),
-                {_, U, D} = read_appup(AppupFile),
-                {U, D};
-            undefined ->
-                %% Fallback to the app.src file, in case the
-                %% application doesn't have a release (useful for the
-                %% apps that live outside the EMQX monorepo):
-                case locate(src, App, ".appup.src") of
             {ok, AppupSrcFile} ->
-                        log("Using ~s as a source of previous update actions~n", [AppupSrcFile]),
+                log("INFO: Using ~s as a source of previous update actions~n", [AppupSrcFile]),
-                        {_, U, D} = read_appup(AppupSrcFile),
+                read_appup(AppupSrcFile);
-                        {U, D};
             undefined ->
+                log("INFO: no appup base found for ~p~n", [App]),
                 {[], []}
-                end
         end,
-    {ensure_version(PrevVersion, Upgrade0), ensure_version(PrevVersion, Downgrade0)}.
+    {ensure_version(PrevVersion, Upgrade), ensure_version(PrevVersion, Downgrade)}.
 
-merge_update_actions(App, Changes, Vsns) ->
+merge_update_actions(App, Changes, Vsns, PrevVersion) ->
     lists:map(fun(Ret = {<<".*">>, _}) ->
                       Ret;
                  ({Vsn, Actions}) ->
+                      case is_skipped_version(App, Vsn, PrevVersion) of
+                          true ->
+                              log("WARN: ~p has version ~s skipped over?~n", [App, Vsn]),
+                              {Vsn, Actions};
+                          false ->
                               {Vsn, do_merge_update_actions(App, Changes, Actions)}
+                      end
               end,
               Vsns).
 
+%% say current version is 1.1.3, and the compare base is version 1.1.1,
+%% but there is a 1.1.2 in appup we may skip merging instructions for
+%% 1.1.2 because it's not used and no way to know what has been changed
+is_skipped_version(App, Vsn, PrevVersion) when is_list(Vsn) andalso is_list(PrevVersion) ->
+    case is_app_external(App) andalso parse_version_number(Vsn) of
+        {ok, VsnTuple} ->
+            case parse_version_number(PrevVersion) of
+                {ok, PrevVsnTuple} ->
+                    VsnTuple > PrevVsnTuple;
+                _ ->
+                    false
+            end;
+        _ ->
+            false
+    end;
+is_skipped_version(_App, _Vsn, _PrevVersion) ->
+    %% if app version is a regexp, we don't know for sure
+    %% return 'false' to be on the safe side
+    false.
+
 do_merge_update_actions(App, {New0, Changed0, Deleted0}, OldActions) ->
     AppSpecific = app_specific_actions(App) -- OldActions,
     AlreadyHandled = lists:flatten(lists:map(fun process_old_action/1, OldActions)),
     New = New0 -- AlreadyHandled,
     Changed = Changed0 -- AlreadyHandled,
     Deleted = Deleted0 -- AlreadyHandled,
-    Reloads = [{load_module, M, brutal_purge, soft_purge, []}
+    HasRestart = contains_restart_application(App, OldActions),
-               || not contains_restart_application(App, OldActions),
+    Actions =
-                  M <- Changed ++ New],
+        case HasRestart of
+            true ->
+                [];
+            false ->
+                [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed] ++
+                [{add_module, M} || M <- New]
+        end,
     {OldActionsWithStop, OldActionsAfterStop} =
         find_application_stop_instruction(App, OldActions),
     OldActionsWithStop ++
-        Reloads ++
+        Actions ++
         OldActionsAfterStop ++
-        [{delete_module, M} || M <- Deleted] ++
+        case HasRestart of
+            true ->
+                [];
+            false ->
+                [{delete_module, M} || M <- Deleted]
+        end ++
         AppSpecific.
 
 %% If an entry restarts an application, there's no need to use
@@ -358,8 +347,12 @@ find_application_stop_instruction(Application, Actions) ->
 %% already handled
 process_old_action({purge, Modules}) ->
     Modules;
+process_old_action({add_module, Module}) ->
+    [Module];
 process_old_action({delete_module, Module}) ->
     [Module];
+process_old_action({update, Module, _Change}) ->
+    [Module];
 process_old_action(LoadModule) when is_tuple(LoadModule) andalso
                                     element(1, LoadModule) =:= load_module ->
     element(2, LoadModule);
@@ -420,11 +413,19 @@ vsn_number_to_string({Major, Minor, Patch}) ->
 
 read_appup(File) ->
     %% NOTE: appup file is a script, it may contain variables or functions.
+   case do_read_appup(File) of
+       {ok, {U, D}} -> {U, D};
+       {error, Reason} -> fail("Failed to parse appup file ~p~n~p", [File, Reason])
+    end.
+
+do_read_appup(File) ->
     case file:script(File, [{'VSN', "VSN"}]) of
-        {ok, Terms} ->
+        {ok, {_, U, D}} ->
-            Terms;
+            {ok, {U, D}};
-        Error ->
+        {ok, Other} ->
-            fail("Failed to parse appup file ~s: ~p", [File, Error])
+            {error, {bad_appup_format, Other}};
+        {error, Reason} ->
+            {error, Reason}
     end.
 
 check_appup_files() ->
@@ -439,52 +440,77 @@ update_appups(Changes) ->
       Changes).
 
 do_update_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) ->
-    case locate(src, App, ".appup.src") of
+    case locate_current_src(App, ".appup.src") of
         {ok, AppupFile} ->
             case contains_contents(AppupFile, Upgrade, Downgrade) of
                 true ->
                     ok;
                 false ->
-                    render_appfile(AppupFile, Upgrade, Downgrade)
+                    render_appup(App, AppupFile, Upgrade, Downgrade)
             end;
         undefined ->
+            maybe_create_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade)
+    end.
+
+maybe_create_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) ->
     case create_stub(App) of
         {ok, AppupFile} ->
-                    render_appfile(AppupFile, Upgrade, Downgrade);
+            render_appup(App, AppupFile, Upgrade, Downgrade);
-                false ->
+        external ->
-                    case parse_appup_diffs(Upgrade, OldUpgrade,
+            %% for external appup, the best we can do is to validate it
-                                           Downgrade, OldDowngrade) of
+            _ = check_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade),
+            ok
+    end.
+
+check_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) ->
+    case parse_appup_diffs(Upgrade, OldUpgrade, Downgrade, OldDowngrade) of
         ok ->
             %% no diff for external dependency; ignore
             ok;
         {diffs, Diffs} ->
             set_invalid(),
-                            log("ERROR: Appup file for the external dependency '~p' is not complete.~n       Missing changes: ~100p~n", [App, Diffs]),
+            log("ERROR: Appup file for '~p' is not complete.~n"
-                            log("NOTE: Some changes above might be already covered by regexes.~n")
+                "Missing:~100p~n", [App, Diffs]),
-                    end
+            notok
-            end
     end.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% Appup file creation
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-render_appfile(File, Upgrade, Downgrade) ->
+render_appup(App, File, Up, Down) ->
-    IOList = io_lib:format("%% -*- mode: erlang -*-\n{VSN,~n  ~p,~n  ~p}.~n", [Upgrade, Downgrade]),
+    IsCheck = getopt(check),
+    case do_read_appup(File) of
+        {ok, {U, D}} when U =:= Up andalso D =:= Down ->
+            ok;
+        {ok, {OldU, OldD}} when IsCheck ->
+            check_appup(App, Up, Down, OldU, OldD);
+        {ok, {_, _}} ->
+            do_render_appup(File, Up, Down);
+        {error, enoent} when IsCheck ->
+            %% failed to read old file, exit
+            log("ERROR: ~s is missing", [File]),
+            set_invalid()
+    end.
+
+do_render_appup(File, Up, Down) ->
+    IOList = io_lib:format("%% -*- mode: erlang -*-~n"
+                           "%% Unless you know what you are doing, DO NOT edit manually!!~n"
+                           "{VSN,~n  ~p,~n  ~p}.~n", [Up, Down]),
     ok = file:write_file(File, IOList).
 
 create_stub(App) ->
     Ext = ".app.src",
-    case locate(src, App, Ext) of
+    case locate_current_src(App, Ext) of
         {ok, AppSrc} ->
             DirName = filename:dirname(AppSrc),
             AppupFile = filename:basename(AppSrc, Ext) ++ ".appup.src",
             Default = {<<".*">>, []},
             AppupFileFullpath = filename:join(DirName, AppupFile),
-            render_appfile(AppupFileFullpath, [Default], [Default]),
+            render_appup(App, AppupFileFullpath, [Default], [Default]),
             {ok, AppupFileFullpath};
         undefined ->
-            false
+            external
     end.
 
 %% we check whether the destination file already has the contents we
@@ -503,8 +529,11 @@ contains_contents(File, Upgrade, Downgrade) ->
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 index_apps(ReleaseDir) ->
-    Apps0 = maps:from_list([index_app(filename:join(ReleaseDir, AppFile)) ||
+    log("INFO: indexing apps in ~s~n", [ReleaseDir]),
-                               AppFile <- filelib:wildcard("**/ebin/*.app", ReleaseDir)]),
+    AppFiles0 = filelib:wildcard("**/ebin/*.app", ReleaseDir),
+    %% everything in _build sub-dir e.g. cuttlefish/_build should be ignored
+    AppFiles = lists:filter(fun(File) -> re:run(File, "_build") =:= nomatch end, AppFiles0),
+    Apps0 = maps:from_list([index_app(filename:join(ReleaseDir, AppFile)) || AppFile <- AppFiles]),
     maps:without(ignored_apps(), Apps0).
 
 index_app(AppFile) ->
@@ -517,7 +546,7 @@ index_app(AppFile) ->
               , modules       = Modules
               }}.
 
-diff_app(App,
+diff_app(UpOrDown, App,
          #app{version = NewVersion, modules = NewModules},
          #app{version = OldVersion, modules = OldModules}) ->
     {New, Changed} =
@@ -535,18 +564,32 @@ diff_app(App,
                  , NewModules
                  ),
     Deleted = maps:keys(maps:without(maps:keys(NewModules), OldModules)),
-    NChanges = length(New) + length(Changed) + length(Deleted),
+    Changes = lists:filter(fun({_T, L}) -> length(L) > 0 end,
-    if NewVersion =:= OldVersion andalso NChanges > 0 ->
+                           [{added, New}, {changed, Changed}, {deleted, Deleted}]),
-            set_invalid(),
+    case NewVersion =:= OldVersion of
-            log("ERROR: Application '~p' contains changes, but its version is not updated~n", [App]);
+        true when Changes =:= [] ->
-       NewVersion > OldVersion ->
+            %% no change
-            log("INFO: Application '~p' has been updated: ~p -> ~p~n", [App, OldVersion, NewVersion]),
             ok;
         true ->
+            set_invalid(),
+            case UpOrDown =:= up of
+                true ->
+                    %% only log for the upgrade case because it would be the same result
+                    log("ERROR: Application '~p' contains changes, but its version is not updated. ~s",
+                        [App, format_changes(Changes)]);
+                false ->
+                    ok
+            end;
+        false ->
+            log("INFO: Application '~p' has been updated: ~p --[~p]--> ~p~n", [App, OldVersion, UpOrDown, NewVersion]),
+            log("INFO: changes [~p]: ~p~n", [UpOrDown, Changes]),
             ok
     end,
     {New, Changed, Deleted}.
 
+format_changes(Changes) ->
+    lists:map(fun({Tag, List}) -> io_lib:format("~p: ~p~n", [Tag, List]) end, Changes).
+
 -spec hashsums(file:filename()) -> #{module() => binary()}.
 hashsums(EbinDir) ->
     maps:from_list(lists:map(
@@ -560,7 +603,7 @@ hashsums(EbinDir) ->
 
 is_app_external(App) ->
     Ext = ".app.src",
-    case locate(src, App, Ext) of
+    case locate_current_src(App, Ext) of
         {ok, _} ->
             false;
         undefined ->
@@ -576,8 +619,16 @@ init_globals(Options) ->
     ets:insert(globals, {valid, true}),
     ets:insert(globals, {options, Options}).
 
+putopt(Option, Value) ->
+    ets:insert(globals, {{option, Option}, Value}).
+
 getopt(Option) ->
-    maps:get(Option, ets:lookup_element(globals, options, 2)).
+    case ets:lookup(globals, {option, Option}) of
+        [] ->
+            maps:get(Option, ets:lookup_element(globals, options, 2));
+        [{_, V}] ->
+            V
+    end.
 
 %% Set a global flag that something about the appfiles is invalid
 set_invalid() ->
@@ -590,33 +641,48 @@ is_valid() ->
 %% Utility functions
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-%% Locate a file in a specified application
+locate_appup(App) ->
-locate(ebin_current, App, Suffix) ->
+    case locate_current_rel(App, ".appup.src") of
-    ReleaseDir = getopt(beams_dir),
+        {ok, File} ->
-    AppStr = atom_to_list(App),
-    case filelib:wildcard(ReleaseDir ++ "/**/ebin/" ++ AppStr ++ Suffix) of
-        [File] ->
             {ok, File};
-        [] ->
+        undefined ->
-            undefined
+            %% fallback to .appup
-    end;
+            locate_current_rel(App, ".appup")
-locate(src, App, Suffix) ->
-    AppStr = atom_to_list(App),
-    SrcDirs = getopt(src_dirs),
-    case filelib:wildcard(SrcDirs ++ AppStr ++ Suffix) of
-        [File] ->
-            {ok, File};
-        [] ->
-            undefined
     end.
 
+locate_current_rel(App, Suffix) ->
+    CurDir = getopt(beams_dir),
+    do_locate(filename:join([CurDir, "**"]), App, Suffix).
+
+%% Locate a file in a specified application
+locate_current_src(App, Suffix) ->
+    SrcDirs = getopt(src_dirs),
+    do_locate(SrcDirs, App, Suffix).
+
+do_locate(Dir, App, Suffix) ->
+    AppStr = atom_to_list(App),
+    Pattern = filename:join(Dir, AppStr ++ Suffix),
+    case find_app(Pattern) of
+        [File] ->
+            {ok, File};
+        [] ->
+            undefined;
+        Files ->
+            error({more_than_one_app_found, Files})
+    end.
+
+find_app(Pattern) ->
+    lists:filter(fun(D) -> re:run(D, "apps/.*/_build") =:= nomatch end,
+                 filelib:wildcard(Pattern)).
+
+bash(undefined) -> ok;
 bash(Script) ->
     bash(Script, []).
 
 bash(Script, Env) ->
     log("+ ~s~n+ Env: ~p~n", [Script, Env]),
     case cmd("bash", #{args => ["-c", Script], env => Env}) of
-        0 -> true;
+        0 -> ok;
         _ -> fail("Failed to run command: ~s", [Script])
     end.