From c3237b6281db2576d2f40aae39ee3e01119cc24b Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 24 Mar 2022 10:39:54 +0800 Subject: [PATCH 01/10] fix(rule): connection test when creating a resource --- .../src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 8 ++- .../emqx_rule_engine/src/emqx_rule_engine.erl | 49 ++++++++++++++----- 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index cd57630b4..f0d73e581 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.8"}, % strict semver, bump manually! + {vsn, "4.3.9"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 0ed0fee3a..010171ab2 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,7 +1,9 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.7", + [{"4.3.8", + [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, + {"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -78,7 +80,9 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.7", + [{"4.3.8", + [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, + {"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index c8e69a17f..1a5fecea1 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -314,24 +314,49 @@ start_resource(ResId) -> end. -spec(test_resource(#{type := _, config := _, _ => _}) -> ok | {error, Reason :: term()}). -test_resource(#{type := Type, config := Config0}) -> +test_resource(#{type := Type} = Params) -> case emqx_rule_registry:find_resource_type(Type) of - {ok, #resource_type{on_create = {ModC, Create}, - on_destroy = {ModD, Destroy}, - params_spec = ParamSpec}} -> - Config = emqx_rule_validator:validate_params(Config0, ParamSpec), - ResId = resource_id(), - try - _ = ?CLUSTER_CALL(init_resource, [ModC, Create, ResId, Config]), - _ = ?CLUSTER_CALL(clear_resource, [ModD, Destroy, ResId]), - ok - catch - throw:Reason -> {error, Reason} + {ok, #resource_type{}} -> + ResId = maps:get(id, Params, resource_id()), + CreateFun = fun() -> _ = create_resource(Params) end, + StatusFun = + fun() -> + case get_resource_status(ResId) of + {ok, #{is_alive := true}} -> + ignore; + {ok, #{is_alive := false}} -> + error(not_alive); + {error, R} -> + error(R) + end + end, + DeleteFun = fun() -> _ = ?CLUSTER_CALL(delete_resource, [ResId]) end, + case + %% create error or status failed + (ok == safe_test_resource(CreateFun, create_resource)) + andalso + safe_test_resource(StatusFun, get_resource_status) + of + ok -> + _ = safe_test_resource(DeleteFun, delete_resource), + ok; + _ -> + _ = safe_test_resource(DeleteFun, delete_resource), + {error, {resource_error, not_available}} end; not_found -> {error, {resource_type_not_found, Type}} end. +safe_test_resource(Fun, ErrorLogInfo) -> + try + _ = Fun(), + ok + catch E:R:S -> + ?LOG(warning, "safe exec fun error, ~0p, ~0p:~0p ~0p", [ErrorLogInfo, E, R, S]), + {error, R} + end. + -spec(get_resource_status(resource_id()) -> {ok, resource_status()} | {error, Reason :: term()}). get_resource_status(ResId) -> case emqx_rule_registry:find_resource_params(ResId) of From 6ca0e635998598db0a815f7470af5e5bbd2c872e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 25 Mar 2022 21:13:55 +0100 Subject: [PATCH 02/10] chore: fix appup update script so it can handle switch between tags --- scripts/update-appup.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh index 7c8ef1925..d21791cce 100755 --- a/scripts/update-appup.sh +++ b/scripts/update-appup.sh @@ -16,12 +16,15 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." PROFILE="${1:-}" case "$PROFILE" in emqx-ee) + GIT_REPO='emqx/emqx-enterprise.git' TAG_PREFIX='e' ;; emqx) + GIT_REPO='emqx/emqx.git' TAG_PREFIX='v' ;; emqx-edge) + GIT_REPO='emqx/emqx.git' TAG_PREFIX='v' ;; *) @@ -36,7 +39,7 @@ esac ## e4.3.11 ## rel-v4.4.3 ## rel-e4.4.3 -PREV_TAG="$(git describe --tag --abbrev=0 --match "[${TAG_PREFIX}|rel-]*" --exclude '*rc*' --exclude '*alpha*' --exclude '*beta*')" +PREV_TAG="${PREV_TAG:-$(git describe --tag --abbrev=0 --match "[${TAG_PREFIX}|rel-]*" --exclude '*rc*' --exclude '*alpha*' --exclude '*beta*')}" shift 1 # bash 3.2 treat empty array as unbound, so we can't use 'ESCRIPT_ARGS=()' here, @@ -77,14 +80,17 @@ fi PREV_DIR_BASE="/tmp/emqx-appup-build" mkdir -p "${PREV_DIR_BASE}" -if [ ! -d "${PREV_TAG}" ]; then +if [ ! -d "${PREV_DIR_BASE}/${PREV_TAG}" ]; then cp -R . "${PREV_DIR_BASE}/${PREV_TAG}" fi pushd "${PREV_DIR_BASE}/${PREV_TAG}" +REMOTE="$(git remote -v | grep "${GIT_REPO}" | head -1 | awk '{print $1}')" git reset --hard -git checkout "${PREV_TAG}" +git clean -fdx +git fetch "$REMOTE" make clean-all +git checkout "${PREV_TAG}" make "$PROFILE" popd From 59fae1714beeec680af8ea725081bd80600c46f5 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 28 Mar 2022 13:53:28 +0200 Subject: [PATCH 03/10] chore: use shorter path --- scripts/update-appup.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh index d21791cce..d5f3b5407 100755 --- a/scripts/update-appup.sh +++ b/scripts/update-appup.sh @@ -78,18 +78,22 @@ if [ "${SKIP_BUILD:-}" != 'yes' ]; then make "${PROFILE}" fi -PREV_DIR_BASE="/tmp/emqx-appup-build" +PREV_DIR_BASE="/tmp/_w" mkdir -p "${PREV_DIR_BASE}" if [ ! -d "${PREV_DIR_BASE}/${PREV_TAG}" ]; then cp -R . "${PREV_DIR_BASE}/${PREV_TAG}" + # always 'yes' in CI + NEW_COPY='yes' +else + NEW_COPY='no' fi pushd "${PREV_DIR_BASE}/${PREV_TAG}" -REMOTE="$(git remote -v | grep "${GIT_REPO}" | head -1 | awk '{print $1}')" -git reset --hard +if [ "$NEW_COPY" = 'no' ]; then + REMOTE="$(git remote -v | grep "${GIT_REPO}" | head -1 | awk '{print $1}')" + git fetch "$REMOTE" +fi git clean -fdx -git fetch "$REMOTE" -make clean-all git checkout "${PREV_TAG}" make "$PROFILE" popd From 46cfcf662e7f1b06846f576be0e996a2a834228b Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 29 Mar 2022 11:24:13 +0800 Subject: [PATCH 04/10] fix(rule): safe apply & test resource in cluster --- .../src/emqx_rule_engine.appup.src | 6 ++- .../emqx_rule_engine/src/emqx_rule_engine.erl | 53 ++++++++----------- .../src/emqx_rule_engine_api.erl | 10 +--- 3 files changed, 26 insertions(+), 43 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 010171ab2..76af48e8f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,7 +2,8 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.8", - [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, @@ -81,7 +82,8 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.8", - [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 1a5fecea1..6c50aca2f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -36,6 +36,7 @@ , create_resource/1 , test_resource/1 , start_resource/1 + , is_source_alive/1 , get_resource_status/1 , get_resource_params/1 , delete_resource/1 @@ -318,45 +319,33 @@ test_resource(#{type := Type} = Params) -> case emqx_rule_registry:find_resource_type(Type) of {ok, #resource_type{}} -> ResId = maps:get(id, Params, resource_id()), - CreateFun = fun() -> _ = create_resource(Params) end, - StatusFun = - fun() -> - case get_resource_status(ResId) of - {ok, #{is_alive := true}} -> - ignore; - {ok, #{is_alive := false}} -> - error(not_alive); - {error, R} -> - error(R) - end - end, - DeleteFun = fun() -> _ = ?CLUSTER_CALL(delete_resource, [ResId]) end, - case - %% create error or status failed - (ok == safe_test_resource(CreateFun, create_resource)) - andalso - safe_test_resource(StatusFun, get_resource_status) - of - ok -> - _ = safe_test_resource(DeleteFun, delete_resource), - ok; - _ -> - _ = safe_test_resource(DeleteFun, delete_resource), - {error, {resource_error, not_available}} + try + _ = create_resource(maps:put(id, ResId, Params)), + true = is_source_alive(ResId), + ok + catch E:R:S -> + ?LOG(warning, "test resource failed, ~0p:~0p ~0p", [E, R, S]), + {error, R} + after + _ = ?CLUSTER_CALL(delete_resource, [ResId]) end; not_found -> {error, {resource_type_not_found, Type}} end. -safe_test_resource(Fun, ErrorLogInfo) -> - try - _ = Fun(), - ok - catch E:R:S -> - ?LOG(warning, "safe exec fun error, ~0p, ~0p:~0p ~0p", [ErrorLogInfo, E, R, S]), - {error, R} +is_source_alive(ResId) -> + case rpc:multicall(ekka_mnesia:running_nodes(), ?MODULE, get_resource_status, [ResId], 5000) of + {ResL, []} -> + is_source_alive_(ResL); + {_, _Errors} -> + false end. +is_source_alive_([]) -> true; +is_source_alive_([{ok, #{is_alive := true}} | ResL]) -> is_source_alive_(ResL); +is_source_alive_([{ok, #{is_alive := false}} | _ResL]) -> false; +is_source_alive_([_Error | _ResL]) -> false. + -spec(get_resource_status(resource_id()) -> {ok, resource_status()} | {error, Reason :: term()}). get_resource_status(ResId) -> case emqx_rule_registry:find_resource_params(ResId) of diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 39ac1e9c2..313591ff3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -296,7 +296,7 @@ do_create_resource(Create, ParsedParams) -> list_resources(#{}, _Params) -> Data0 = lists:foldr(fun maybe_record_to_map/2, [], emqx_rule_registry:get_resources()), Data = lists:map(fun(Res = #{id := ResId}) -> - Status = get_aggregated_status(ResId), + Status = emqx_rule_engine:is_source_alive(ResId), maps:put(status, Status, Res) end, Data0), return({ok, Data}). @@ -304,14 +304,6 @@ list_resources(#{}, _Params) -> list_resources_by_type(#{type := Type}, _Params) -> return_all(emqx_rule_registry:get_resources_by_type(Type)). -get_aggregated_status(ResId) -> - lists:all(fun(Node) -> - case rpc:call(Node, emqx_rule_engine, get_resource_status, [ResId]) of - {ok, #{is_alive := true}} -> true; - _ -> false - end - end, ekka_mnesia:running_nodes()). - show_resource(#{id := Id}, _Params) -> case emqx_rule_registry:find_resource(Id) of {ok, R} -> From f9a85614d4b287ba5f5c32a0d280faa11a59142a Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 28 Mar 2022 16:48:19 +0800 Subject: [PATCH 05/10] chore: export test function for ee's hot-conf --- src/emqx.appup.src | 151 +++++++++++++++++++++++++++++++++------- src/emqx_cm_locker.erl | 6 +- src/emqx_flapping.erl | 6 +- src/emqx_limiter.erl | 27 ++++--- src/emqx_listeners.erl | 15 ++++ src/emqx_shared_sub.erl | 2 +- 6 files changed, 167 insertions(+), 40 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 86ca424b3..138bd495c 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,8 +1,24 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, +<<<<<<< HEAD [{"4.3.13", [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, +======= + [{"4.3.14", + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]} + ]}, + {"4.3.13", + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, +>>>>>>> 3b81db7ae (chore: export test function for ee's hot-conf) {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -16,7 +32,11 @@ {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -39,7 +59,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, @@ -64,7 +87,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -89,7 +115,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -118,7 +147,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -147,7 +179,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -177,7 +212,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -207,7 +245,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -238,7 +279,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -270,7 +313,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -303,7 +348,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -336,7 +383,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", - [{load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, @@ -372,7 +421,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, @@ -411,8 +462,22 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], +<<<<<<< HEAD [{"4.3.13", [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, +======= + [{"4.3.14", + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]} + ]}, + {"4.3.13", + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, +>>>>>>> 3b81db7ae (chore: export test function for ee's hot-conf) {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -426,7 +491,10 @@ {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, @@ -448,7 +516,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -472,7 +543,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -496,7 +570,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -524,7 +601,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -552,7 +632,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -581,7 +664,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -610,7 +696,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -640,7 +729,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -671,7 +762,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -703,7 +796,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -735,7 +830,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", - [{load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, @@ -770,7 +867,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_cm_locker.erl b/src/emqx_cm_locker.erl index 4ce6a9279..5fb35dbbf 100644 --- a/src/emqx_cm_locker.erl +++ b/src/emqx_cm_locker.erl @@ -28,6 +28,11 @@ , unlock/1 ]). +%% for testing +-ifdef(TEST). +-export([strategy/0]). +-endif. + -spec(start_link() -> startlink_ret()). start_link() -> ekka_locker:start_link(?MODULE). @@ -63,4 +68,3 @@ unlock(ClientId) -> -spec(strategy() -> local | leader | quorum | all). strategy() -> emqx:get_env(session_locking_strategy, quorum). - diff --git a/src/emqx_flapping.erl b/src/emqx_flapping.erl index a0eab9c18..6ce45e962 100644 --- a/src/emqx_flapping.erl +++ b/src/emqx_flapping.erl @@ -38,6 +38,10 @@ , code_change/3 ]). +%% for testing +-ifdef(TEST). +-export([get_policy/0]). +-endif. %% Tab -define(FLAPPING_TAB, ?MODULE). %% Default Policy @@ -160,4 +164,4 @@ code_change(_OldVsn, State, _Extra) -> ensure_timer(State) -> Timeout = maps:get(duration, get_policy()), TRef = emqx_misc:start_timer(Timeout, expired_detecting), - State#{expired_timer => TRef}. \ No newline at end of file + State#{expired_timer => TRef}. diff --git a/src/emqx_limiter.erl b/src/emqx_limiter.erl index 52b830c8e..1cca7140b 100644 --- a/src/emqx_limiter.erl +++ b/src/emqx_limiter.erl @@ -23,7 +23,8 @@ , init/4 %% XXX: Compatible with before 4.2 version , info/1 , check/2 - , update_overall_limiter/4 + , update_overall_limiter/3 + , delete_overall_limiter/1 ]). -record(limiter, { @@ -154,14 +155,18 @@ is_message_limiter(conn_messages_routing) -> true; is_message_limiter(overall_messages_routing) -> true; is_message_limiter(_) -> false. -update_overall_limiter(Zone, Name, Capacity, Interval) -> - case is_overall_limiter(Name) of - false -> false; - _ -> - try - esockd_limiter:update({Zone, Name}, Capacity, Interval), - true - catch _:_:_ -> - false - end +update_overall_limiter(Zone, Capacity, Interval) -> + try + esockd_limiter:update({Zone, overall_messages_routing}, Capacity, Interval), + true + catch _:_:_ -> + false + end. + +delete_overall_limiter(Zone) -> + try + esockd_limiter:delete({Zone, overall_messages_routing}), + true + catch _:_:_ -> + false end. diff --git a/src/emqx_listeners.erl b/src/emqx_listeners.erl index 1f3d1776b..c509adb36 100644 --- a/src/emqx_listeners.erl +++ b/src/emqx_listeners.erl @@ -29,6 +29,7 @@ -export([ start_listener/1 , start_listener/3 , stop_listener/1 + , update_listeners_env/2 , restart_listener/1 , restart_listener/3 ]). @@ -187,6 +188,20 @@ with_port(Port, Opts = #{socket_opts := SocketOption}) when is_integer(Port) -> with_port({Addr, Port}, Opts = #{socket_opts := SocketOption}) -> Opts#{socket_opts => [{ip, Addr}, {port, Port}| SocketOption]}. +update_listeners_env(Action, NewConf = #{name := NewName, proto := NewProto}) -> + Listener = emqx:get_env(listeners, []), + Listener1 = lists:filter( + fun(#{name := Name, proto := Proto}) -> + not (Name =:= NewName andalso Proto =:= NewProto) + end, Listener), + Listener2 = + case Action of + update -> [NewConf | Listener1]; + delete -> Listener1 + end, + application:set_env(emqx, listeners, Listener2), + ok. + %% @doc Restart all listeners -spec(restart() -> ok). restart() -> diff --git a/src/emqx_shared_sub.erl b/src/emqx_shared_sub.erl index 97aa778f3..2de15eb55 100644 --- a/src/emqx_shared_sub.erl +++ b/src/emqx_shared_sub.erl @@ -47,7 +47,7 @@ ]). %% for testing --export([subscribers/2]). +-export([subscribers/2, ack_enabled/0]). %% gen_server callbacks -export([ init/1 From 517f8a7c9b9512b88215b8f54fedf4f7ead6d25f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 28 Mar 2022 16:47:49 +0800 Subject: [PATCH 06/10] chore: add validator schema function --- priv/emqx.schema | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/priv/emqx.schema b/priv/emqx.schema index b6a9ee41e..cfd3fa337 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -258,6 +258,28 @@ end}. {validator, "range4ports", "must be 1024 to 134217727", fun(X) -> X >= 1024 andalso X =< 134217727 end}. +{validator, "range:0-2", "must be 0 to 2", + fun(X) -> X >= 0 andalso X =< 2 end}. + +{validator, "range:0-128", "must be 0 to 128", + fun(X) -> X >= 0 andalso X =< 128 end}. + +{validator, "range:0-65535", "must be 0 to 65535", + fun(X) -> X >= 0 andalso X =< 65535 end}. + +{validator, "range:1-65535", "must be 1 to 65535", + fun(X) -> X >= 1 andalso X =< 65535 end}. + +{validator, "range:1-9", "must be 1 to 9", + fun(X) -> X >= 1 andalso X =< 9 end}. + +{validator, "range:8-15", "must be 8 to 15", + fun(X) -> X >= 8 andalso X =< 15 end}. + +{validator, "range:0-1024", "must be 0 to 1024", + fun(X) -> X >= 0 andalso X =< 1024 end}. + + %% @doc http://www.erlang.org/doc/man/erl.html#%2bzdbbl {mapping, "node.dist_buffer_size", "vm_args.+zdbbl", [ {datatype, bytesize}, @@ -293,10 +315,10 @@ end}. {default, 1000}, {datatype, integer}, hidden, - {validators, ["positive_integer"]} + {validators, ["range:0-inf"]} ]}. -{validator, "positive_integer", "must be a positive integer", +{validator, "range:0-inf", "must be a non neg_integer", fun(X) -> X >= 0 end}. %% Note: OTP R15 and earlier uses -env ERL_MAX_ETS_TABLES, @@ -801,7 +823,8 @@ end}. %% @doc Set the Maximum topic levels. {mapping, "mqtt.max_topic_levels", "emqx.max_topic_levels", [ {default, 128}, - {datatype, integer} + {datatype, integer}, + {validators, ["range:0-inf"]} ]}. %% @doc Set the Maximum QoS allowed. @@ -814,7 +837,8 @@ end}. %% @doc Set the Maximum Topic Alias. {mapping, "mqtt.max_topic_alias", "emqx.max_topic_alias", [ {default, 65535}, - {datatype, integer} + {datatype, integer}, + {validators, ["range:0-65535"]} ]}. %% @doc Whether the server supports MQTT retained messages. @@ -911,7 +935,8 @@ end}. %% @doc Set the Maximum topic levels. {mapping, "zone.$name.max_topic_levels", "emqx.zones", [ - {datatype, integer} + {datatype, integer}, + {validators, ["range:0-128"]} ]}. %% @doc Set the Maximum QoS allowed. From d297e883c24d40dda582a8d8a4803b70d0d493d7 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 29 Mar 2022 17:37:00 +0800 Subject: [PATCH 07/10] fix: appup --- apps/emqx_rule_engine/src/emqx_rule_engine.appup.src | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 76af48e8f..98ca43579 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,7 +2,10 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.8", - [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, @@ -82,7 +85,10 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.8", - [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, From 47e807b2ac8cf3b6b837ebd42d47fb09ebe1b63b Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 29 Mar 2022 19:16:06 +0800 Subject: [PATCH 08/10] Revert "fix(rule): connection test when creating a resource" --- .../src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 16 ++------- .../emqx_rule_engine/src/emqx_rule_engine.erl | 34 ++++++------------- .../src/emqx_rule_engine_api.erl | 10 +++++- 4 files changed, 22 insertions(+), 40 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index f0d73e581..cd57630b4 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.9"}, % strict semver, bump manually! + {vsn, "4.3.8"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 98ca43579..0ed0fee3a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,13 +1,7 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.8", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {"4.3.7", + [{"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -84,13 +78,7 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.8", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {"4.3.7", + [{"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 6c50aca2f..c8e69a17f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -36,7 +36,6 @@ , create_resource/1 , test_resource/1 , start_resource/1 - , is_source_alive/1 , get_resource_status/1 , get_resource_params/1 , delete_resource/1 @@ -315,37 +314,24 @@ start_resource(ResId) -> end. -spec(test_resource(#{type := _, config := _, _ => _}) -> ok | {error, Reason :: term()}). -test_resource(#{type := Type} = Params) -> +test_resource(#{type := Type, config := Config0}) -> case emqx_rule_registry:find_resource_type(Type) of - {ok, #resource_type{}} -> - ResId = maps:get(id, Params, resource_id()), + {ok, #resource_type{on_create = {ModC, Create}, + on_destroy = {ModD, Destroy}, + params_spec = ParamSpec}} -> + Config = emqx_rule_validator:validate_params(Config0, ParamSpec), + ResId = resource_id(), try - _ = create_resource(maps:put(id, ResId, Params)), - true = is_source_alive(ResId), + _ = ?CLUSTER_CALL(init_resource, [ModC, Create, ResId, Config]), + _ = ?CLUSTER_CALL(clear_resource, [ModD, Destroy, ResId]), ok - catch E:R:S -> - ?LOG(warning, "test resource failed, ~0p:~0p ~0p", [E, R, S]), - {error, R} - after - _ = ?CLUSTER_CALL(delete_resource, [ResId]) + catch + throw:Reason -> {error, Reason} end; not_found -> {error, {resource_type_not_found, Type}} end. -is_source_alive(ResId) -> - case rpc:multicall(ekka_mnesia:running_nodes(), ?MODULE, get_resource_status, [ResId], 5000) of - {ResL, []} -> - is_source_alive_(ResL); - {_, _Errors} -> - false - end. - -is_source_alive_([]) -> true; -is_source_alive_([{ok, #{is_alive := true}} | ResL]) -> is_source_alive_(ResL); -is_source_alive_([{ok, #{is_alive := false}} | _ResL]) -> false; -is_source_alive_([_Error | _ResL]) -> false. - -spec(get_resource_status(resource_id()) -> {ok, resource_status()} | {error, Reason :: term()}). get_resource_status(ResId) -> case emqx_rule_registry:find_resource_params(ResId) of diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 313591ff3..39ac1e9c2 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -296,7 +296,7 @@ do_create_resource(Create, ParsedParams) -> list_resources(#{}, _Params) -> Data0 = lists:foldr(fun maybe_record_to_map/2, [], emqx_rule_registry:get_resources()), Data = lists:map(fun(Res = #{id := ResId}) -> - Status = emqx_rule_engine:is_source_alive(ResId), + Status = get_aggregated_status(ResId), maps:put(status, Status, Res) end, Data0), return({ok, Data}). @@ -304,6 +304,14 @@ list_resources(#{}, _Params) -> list_resources_by_type(#{type := Type}, _Params) -> return_all(emqx_rule_registry:get_resources_by_type(Type)). +get_aggregated_status(ResId) -> + lists:all(fun(Node) -> + case rpc:call(Node, emqx_rule_engine, get_resource_status, [ResId]) of + {ok, #{is_alive := true}} -> true; + _ -> false + end + end, ekka_mnesia:running_nodes()). + show_resource(#{id := Id}, _Params) -> case emqx_rule_registry:find_resource(Id) of {ok, R} -> From 16b8fc6edc162efa0a8c568f7e0f6866020736fe Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 29 Mar 2022 15:16:57 +0800 Subject: [PATCH 09/10] fix: emqx.appup.src --- src/emqx.appup.src | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 138bd495c..29c060b78 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,24 +1,12 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, -<<<<<<< HEAD [{"4.3.13", [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, -======= - [{"4.3.14", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, - {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]} - ]}, - {"4.3.13", - [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, ->>>>>>> 3b81db7ae (chore: export test function for ee's hot-conf) {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -33,7 +21,6 @@ {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -462,22 +449,12 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], -<<<<<<< HEAD - [{"4.3.13", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, -======= - [{"4.3.14", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, - {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]} - ]}, - {"4.3.13", + [{"4.3.13", [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, ->>>>>>> 3b81db7ae (chore: export test function for ee's hot-conf) {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, From 0ac979361e3e549972d8d4be8d72efa7c9998409 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 31 Mar 2022 09:31:53 +0200 Subject: [PATCH 10/10] fix(appup): re-generate emqx.appup.src --- src/emqx.appup.src | 102 ++++++++++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 44 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 372e26df7..c793df943 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,33 +2,40 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.1", - [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {add_module,emqx_relup}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, - {"4.4.0", - [{load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {add_module,emqx_relup}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {add_module,emqx_relup}]}, + {"4.4.0", + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {add_module,emqx_relup}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -45,33 +52,39 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.1", - [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, + {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {delete_module,emqx_relup}]}, {"4.4.0", - [{load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {delete_module,emqx_relup}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -84,5 +97,6 @@ {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, + {delete_module,emqx_relup}]}, {<<".*">>,[]}]}.