From 02aa665e543909005d459cfb078985f852038710 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 1 Mar 2022 11:36:56 +0100 Subject: [PATCH 1/6] chore: bump version to prepare the next release --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 197ec2906..bc9c49e57 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.12"}). +-define(EMQX_RELEASE, {opensource, "4.3.13-rc.1"}). -else. From cca9e52c9a0f475f07196e139138e137d4414118 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 1 Mar 2022 11:37:13 +0100 Subject: [PATCH 2/6] chore(appup): add missing appup instructions --- apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src | 12 ++++++++---- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.appup.src | 6 ++++-- src/emqx.appup.src | 10 +++++----- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src index 9ae492c5f..46b3c8312 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src @@ -13,12 +13,14 @@ {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, {<<"4.3.4">>, - [{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, + [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]}, {<<"4.3.5">>, - [{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{<<"4.3.[0-3]">>, [{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]}, @@ -33,10 +35,12 @@ {delete_module,emqx_acl_mnesia_migrator}, {delete_module,emqx_acl_mnesia_db}]}, {<<"4.3.4">>, - [{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, + [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]}, {<<"4.3.5">>, - [{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index dd39cfc40..cd57630b4 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.7"}, % strict semver, bump manually! + {vsn, "4.3.8"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 7a758dcc4..77b41dd4a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,6 +1,7 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.6", + [{"4.3.7",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.3.6", [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {update,emqx_rule_metrics,{advanced,["4.3.6"]}}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, @@ -58,7 +59,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.6", + [{"4.3.7",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.3.6", [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {update,emqx_rule_metrics,{advanced,["4.3.6"]}}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 14ef02f80..c69f104e3 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,6 @@ %% -*- mode: erlang -*- {VSN, - [ - {"4.3.13", + [{"4.3.13", [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -10,6 +9,7 @@ {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -390,9 +390,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [ - {"4.3.13", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{"4.3.13", + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, From 52ff180e55ef48d7971301f8c22693ebd791f48d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 1 Mar 2022 16:07:42 +0100 Subject: [PATCH 3/6] chore: pin esockd 5.8.5 --- apps/emqx_sn/rebar.config | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_sn/rebar.config b/apps/emqx_sn/rebar.config index 5fecbb815..120d4767d 100644 --- a/apps/emqx_sn/rebar.config +++ b/apps/emqx_sn/rebar.config @@ -2,7 +2,7 @@ {plugins, [rebar3_proper]}. {deps, - [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.7.4"}}}, + [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.5"}}}, {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} ]}. diff --git a/rebar.config b/rebar.config index 706c9bd1d..131aafebb 100644 --- a/rebar.config +++ b/rebar.config @@ -42,7 +42,7 @@ , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.4"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.5"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.8"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} From 0996b79d961a0bed649ad32d4fddabc54ba228ff Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 1 Mar 2022 09:47:32 +0100 Subject: [PATCH 4/6] chore: add a script to simplify appup update --- scripts/update-appup.sh | 61 ++++++++++++++++++++++++++++++++++++ scripts/update_appup.escript | 55 ++++++++++++++++++++++---------- 2 files changed, 99 insertions(+), 17 deletions(-) create mode 100755 scripts/update-appup.sh diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh new file mode 100755 index 000000000..e62558a08 --- /dev/null +++ b/scripts/update-appup.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +## This script wrapps update_appup.escript, +## it provides a more commonly used set of default args. + +## Arg1: EMQX PROFILE + +set -euo pipefail + +usage() { + echo "$0 PROFILE PREV_VERSION" +} +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + +PROFILE="${1:-}" +case "$PROFILE" in + emqx-ee) + DIR='enterprise' + TAG_PREFIX='e' + ;; + emqx) + DIR='broker' + TAG_PREFIX='v' + ;; + emqx-edge) + DIR='edge' + TAG_PREFIX='v' + ;; + *) + echo "Unknown profile $PROFILE" + usage + exit 1 + ;; +esac + +PREV_VERSION="$(git describe --tag --match "${TAG_PREFIX}*" | grep -oE "${TAG_PREFIX}4\.[0-9]+\.[0-9]+")" +PREV_VERSION="${PREV_VERSION#[e|v]}" + +shift 1 +ESCRIPT_ARGS="$*" + +SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" +ARCH="${ARCH:-$(uname -m)}" +case "$ARCH" in + x86_64) + ARCH='amd64' + ;; + aarch64) + ARCH='arm64' + ;; + arm*) + ARCH=arm + ;; +esac + +PACKAGE_NAME="${PROFILE}-${SYSTEM}-${PREV_VERSION}-${ARCH}.zip" +DOWNLOAD_URL="https://www.emqx.com/downloads/${DIR}/v${PREV_VERSION}/${PACKAGE_NAME}" + +# shellcheck disable=SC2086 +./scripts/update_appup.escript --binary-rel-url "$DOWNLOAD_URL" $ESCRIPT_ARGS "$PREV_VERSION" diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 965de1efe..5bc709267 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -32,8 +32,8 @@ Options: --make-command A command used to assemble the release --release-dir Release directory --src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/' - --binary-rel-url Binary release URL pattern. %TAG% variable is substituted with the release tag. - E.g. \"https://github.com/emqx/emqx/releases/download/v%TAG%/emqx-centos7-%TAG%-amd64.zip\" + --binary-rel-url Binary release URL pattern. + E.g. https://www.emqx.com/downloads/broker/v4.3.12/emqx-centos7-4.3.12-amd64.zip ". -record(app, @@ -171,9 +171,10 @@ download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) - BaseDir = "/tmp/emqx-baseline-bin/", Dir = filename:basename(Repo, ".git") ++ [$-|Tag], Filename = filename:join(BaseDir, Dir), - Script = "mkdir -p ${OUTFILE} && - wget -c -O ${OUTFILE}.zip ${URL} && - unzip -n -d ${OUTFILE} ${OUTFILE}.zip", + Script = "echo \"Download: ${OUTFILE}\" && + mkdir -p ${OUTFILE} && + curl -f -L -o ${OUTFILE}.zip ${URL} && + unzip -q -n -d ${OUTFILE} ${OUTFILE}.zip", Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}], bash(Script, Env), {ok, Filename}. @@ -208,8 +209,8 @@ find_appup_actions(App, {OldUpgrade0, OldDowngrade0} = find_old_appup_actions(App, PrevVersion), OldUpgrade = ensure_all_patch_versions(App, CurrVersion, OldUpgrade0), OldDowngrade = ensure_all_patch_versions(App, CurrVersion, OldDowngrade0), - Upgrade = merge_update_actions(App, diff_app(App, CurrAppIdx, PrevAppIdx), OldUpgrade), - Downgrade = merge_update_actions(App, diff_app(App, PrevAppIdx, CurrAppIdx), OldDowngrade), + Upgrade = merge_update_actions(App, diff_app(up, App, CurrAppIdx, PrevAppIdx), OldUpgrade), + Downgrade = merge_update_actions(App, diff_app(down, App, PrevAppIdx, CurrAppIdx), OldDowngrade), if OldUpgrade =:= Upgrade andalso OldDowngrade =:= Downgrade -> %% The appup file has been already updated: []; @@ -521,7 +522,7 @@ index_app(AppFile) -> , modules = Modules }}. -diff_app(App, +diff_app(UpOrDown, App, #app{version = NewVersion, modules = NewModules}, #app{version = OldVersion, modules = OldModules}) -> {New, Changed} = @@ -539,18 +540,31 @@ diff_app(App, , NewModules ), Deleted = maps:keys(maps:without(maps:keys(NewModules), OldModules)), - NChanges = length(New) + length(Changed) + length(Deleted), - if NewVersion =:= OldVersion andalso NChanges > 0 -> - set_invalid(), - log("ERROR: Application '~p' contains changes, but its version is not updated~n", [App]); - NewVersion > OldVersion -> - log("INFO: Application '~p' has been updated: ~p -> ~p~n", [App, OldVersion, NewVersion]), + Changes = lists:filter(fun({_T, L}) -> length(L) > 0 end, + [{added, New}, {changed, Changed}, {deleted, Deleted}]), + case NewVersion =:= OldVersion of + true when Changes =:= [] -> + %% no change ok; - true -> + true -> + set_invalid(), + case UpOrDown =:= up of + true -> + %% only log for the upgrade case because it would be the same result + log("ERROR: Application '~p' contains changes, but its version is not updated. ~s", + [App, format_changes(Changes)]); + false -> + ok + end; + false -> + log("INFO: Application '~p' has been updated: ~p --[~p]--> ~p~n", [App, OldVersion, UpOrDown, NewVersion]), ok end, {New, Changed, Deleted}. +format_changes(Changes) -> + lists:map(fun({Tag, List}) -> io_lib:format("~p: ~p~n", [Tag, List]) end, Changes). + -spec hashsums(file:filename()) -> #{module() => binary()}. hashsums(EbinDir) -> maps:from_list(lists:map( @@ -607,13 +621,20 @@ locate(ebin_current, App, Suffix) -> locate(src, App, Suffix) -> AppStr = atom_to_list(App), SrcDirs = getopt(src_dirs), - case filelib:wildcard(SrcDirs ++ AppStr ++ Suffix) of + case find_app(SrcDirs ++ AppStr ++ Suffix) of [File] -> {ok, File}; [] -> - undefined + undefined; + Files -> + error({more_than_one_app_found, Files}) end. +find_app(Pattern) -> + %% exclude _build dir inside apps + lists:filter(fun(S) -> string:find(S, "/_build/") =:= nomatch end, + filelib:wildcard(Pattern)). + bash(Script) -> bash(Script, []). From ef382bf34cbf6c270a4a3ebd63c185a055fb3142 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 1 Mar 2022 15:31:06 +0100 Subject: [PATCH 5/6] chore: add --make-command to update_appup script args --- scripts/update-appup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh index e62558a08..e1ef0c3c0 100755 --- a/scripts/update-appup.sh +++ b/scripts/update-appup.sh @@ -58,4 +58,4 @@ PACKAGE_NAME="${PROFILE}-${SYSTEM}-${PREV_VERSION}-${ARCH}.zip" DOWNLOAD_URL="https://www.emqx.com/downloads/${DIR}/v${PREV_VERSION}/${PACKAGE_NAME}" # shellcheck disable=SC2086 -./scripts/update_appup.escript --binary-rel-url "$DOWNLOAD_URL" $ESCRIPT_ARGS "$PREV_VERSION" +./scripts/update_appup.escript --make-command "make ${PROFILE}-rel" --binary-rel-url "$DOWNLOAD_URL" $ESCRIPT_ARGS "$PREV_VERSION" From 802967074178a9b8a4b789b26f766644415191da Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 1 Mar 2022 14:57:17 +0100 Subject: [PATCH 6/6] ci: add appup update check to ci --- .github/workflows/apps_version_check.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 4975e5c11..67355155f 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -6,7 +6,22 @@ jobs: check_apps_version: runs-on: ubuntu-20.04 + strategy: + matrix: + erl_otp: + - erl23.2.7.2-emqx-3 + os: + - ubuntu20.04 + + container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }} + steps: - uses: actions/checkout@v1 - name: Check apps version run: ./scripts/apps-version-check.sh + - name: Check relup (ce) + if: endsWith(github.repository, 'emqx') + run: ./scripts/update-appup.sh emqx --check + - name: Check relup (ee) + if: endsWith(github.repository, 'enterprise') + run: ./scripts/update-appup.sh emqx-ee --check