From c3237b6281db2576d2f40aae39ee3e01119cc24b Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 24 Mar 2022 10:39:54 +0800 Subject: [PATCH 01/15] fix(rule): connection test when creating a resource --- .../src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 8 ++- .../emqx_rule_engine/src/emqx_rule_engine.erl | 49 ++++++++++++++----- 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index cd57630b4..f0d73e581 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.8"}, % strict semver, bump manually! + {vsn, "4.3.9"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 0ed0fee3a..010171ab2 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,7 +1,9 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.7", + [{"4.3.8", + [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, + {"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -78,7 +80,9 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.7", + [{"4.3.8", + [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, + {"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index c8e69a17f..1a5fecea1 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -314,24 +314,49 @@ start_resource(ResId) -> end. -spec(test_resource(#{type := _, config := _, _ => _}) -> ok | {error, Reason :: term()}). -test_resource(#{type := Type, config := Config0}) -> +test_resource(#{type := Type} = Params) -> case emqx_rule_registry:find_resource_type(Type) of - {ok, #resource_type{on_create = {ModC, Create}, - on_destroy = {ModD, Destroy}, - params_spec = ParamSpec}} -> - Config = emqx_rule_validator:validate_params(Config0, ParamSpec), - ResId = resource_id(), - try - _ = ?CLUSTER_CALL(init_resource, [ModC, Create, ResId, Config]), - _ = ?CLUSTER_CALL(clear_resource, [ModD, Destroy, ResId]), - ok - catch - throw:Reason -> {error, Reason} + {ok, #resource_type{}} -> + ResId = maps:get(id, Params, resource_id()), + CreateFun = fun() -> _ = create_resource(Params) end, + StatusFun = + fun() -> + case get_resource_status(ResId) of + {ok, #{is_alive := true}} -> + ignore; + {ok, #{is_alive := false}} -> + error(not_alive); + {error, R} -> + error(R) + end + end, + DeleteFun = fun() -> _ = ?CLUSTER_CALL(delete_resource, [ResId]) end, + case + %% create error or status failed + (ok == safe_test_resource(CreateFun, create_resource)) + andalso + safe_test_resource(StatusFun, get_resource_status) + of + ok -> + _ = safe_test_resource(DeleteFun, delete_resource), + ok; + _ -> + _ = safe_test_resource(DeleteFun, delete_resource), + {error, {resource_error, not_available}} end; not_found -> {error, {resource_type_not_found, Type}} end. +safe_test_resource(Fun, ErrorLogInfo) -> + try + _ = Fun(), + ok + catch E:R:S -> + ?LOG(warning, "safe exec fun error, ~0p, ~0p:~0p ~0p", [ErrorLogInfo, E, R, S]), + {error, R} + end. + -spec(get_resource_status(resource_id()) -> {ok, resource_status()} | {error, Reason :: term()}). get_resource_status(ResId) -> case emqx_rule_registry:find_resource_params(ResId) of From 46cfcf662e7f1b06846f576be0e996a2a834228b Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 29 Mar 2022 11:24:13 +0800 Subject: [PATCH 02/15] fix(rule): safe apply & test resource in cluster --- .../src/emqx_rule_engine.appup.src | 6 ++- .../emqx_rule_engine/src/emqx_rule_engine.erl | 53 ++++++++----------- .../src/emqx_rule_engine_api.erl | 10 +--- 3 files changed, 26 insertions(+), 43 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 010171ab2..76af48e8f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,7 +2,8 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.8", - [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, @@ -81,7 +82,8 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.8", - [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 1a5fecea1..6c50aca2f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -36,6 +36,7 @@ , create_resource/1 , test_resource/1 , start_resource/1 + , is_source_alive/1 , get_resource_status/1 , get_resource_params/1 , delete_resource/1 @@ -318,45 +319,33 @@ test_resource(#{type := Type} = Params) -> case emqx_rule_registry:find_resource_type(Type) of {ok, #resource_type{}} -> ResId = maps:get(id, Params, resource_id()), - CreateFun = fun() -> _ = create_resource(Params) end, - StatusFun = - fun() -> - case get_resource_status(ResId) of - {ok, #{is_alive := true}} -> - ignore; - {ok, #{is_alive := false}} -> - error(not_alive); - {error, R} -> - error(R) - end - end, - DeleteFun = fun() -> _ = ?CLUSTER_CALL(delete_resource, [ResId]) end, - case - %% create error or status failed - (ok == safe_test_resource(CreateFun, create_resource)) - andalso - safe_test_resource(StatusFun, get_resource_status) - of - ok -> - _ = safe_test_resource(DeleteFun, delete_resource), - ok; - _ -> - _ = safe_test_resource(DeleteFun, delete_resource), - {error, {resource_error, not_available}} + try + _ = create_resource(maps:put(id, ResId, Params)), + true = is_source_alive(ResId), + ok + catch E:R:S -> + ?LOG(warning, "test resource failed, ~0p:~0p ~0p", [E, R, S]), + {error, R} + after + _ = ?CLUSTER_CALL(delete_resource, [ResId]) end; not_found -> {error, {resource_type_not_found, Type}} end. -safe_test_resource(Fun, ErrorLogInfo) -> - try - _ = Fun(), - ok - catch E:R:S -> - ?LOG(warning, "safe exec fun error, ~0p, ~0p:~0p ~0p", [ErrorLogInfo, E, R, S]), - {error, R} +is_source_alive(ResId) -> + case rpc:multicall(ekka_mnesia:running_nodes(), ?MODULE, get_resource_status, [ResId], 5000) of + {ResL, []} -> + is_source_alive_(ResL); + {_, _Errors} -> + false end. +is_source_alive_([]) -> true; +is_source_alive_([{ok, #{is_alive := true}} | ResL]) -> is_source_alive_(ResL); +is_source_alive_([{ok, #{is_alive := false}} | _ResL]) -> false; +is_source_alive_([_Error | _ResL]) -> false. + -spec(get_resource_status(resource_id()) -> {ok, resource_status()} | {error, Reason :: term()}). get_resource_status(ResId) -> case emqx_rule_registry:find_resource_params(ResId) of diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 39ac1e9c2..313591ff3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -296,7 +296,7 @@ do_create_resource(Create, ParsedParams) -> list_resources(#{}, _Params) -> Data0 = lists:foldr(fun maybe_record_to_map/2, [], emqx_rule_registry:get_resources()), Data = lists:map(fun(Res = #{id := ResId}) -> - Status = get_aggregated_status(ResId), + Status = emqx_rule_engine:is_source_alive(ResId), maps:put(status, Status, Res) end, Data0), return({ok, Data}). @@ -304,14 +304,6 @@ list_resources(#{}, _Params) -> list_resources_by_type(#{type := Type}, _Params) -> return_all(emqx_rule_registry:get_resources_by_type(Type)). -get_aggregated_status(ResId) -> - lists:all(fun(Node) -> - case rpc:call(Node, emqx_rule_engine, get_resource_status, [ResId]) of - {ok, #{is_alive := true}} -> true; - _ -> false - end - end, ekka_mnesia:running_nodes()). - show_resource(#{id := Id}, _Params) -> case emqx_rule_registry:find_resource(Id) of {ok, R} -> From d297e883c24d40dda582a8d8a4803b70d0d493d7 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 29 Mar 2022 17:37:00 +0800 Subject: [PATCH 03/15] fix: appup --- apps/emqx_rule_engine/src/emqx_rule_engine.appup.src | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 76af48e8f..98ca43579 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,7 +2,10 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.8", - [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, @@ -82,7 +85,10 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.8", - [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, From 47e807b2ac8cf3b6b837ebd42d47fb09ebe1b63b Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 29 Mar 2022 19:16:06 +0800 Subject: [PATCH 04/15] Revert "fix(rule): connection test when creating a resource" --- .../src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 16 ++------- .../emqx_rule_engine/src/emqx_rule_engine.erl | 34 ++++++------------- .../src/emqx_rule_engine_api.erl | 10 +++++- 4 files changed, 22 insertions(+), 40 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index f0d73e581..cd57630b4 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.9"}, % strict semver, bump manually! + {vsn, "4.3.8"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 98ca43579..0ed0fee3a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,13 +1,7 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.8", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {"4.3.7", + [{"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -84,13 +78,7 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.8", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {"4.3.7", + [{"4.3.7", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 6c50aca2f..c8e69a17f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -36,7 +36,6 @@ , create_resource/1 , test_resource/1 , start_resource/1 - , is_source_alive/1 , get_resource_status/1 , get_resource_params/1 , delete_resource/1 @@ -315,37 +314,24 @@ start_resource(ResId) -> end. -spec(test_resource(#{type := _, config := _, _ => _}) -> ok | {error, Reason :: term()}). -test_resource(#{type := Type} = Params) -> +test_resource(#{type := Type, config := Config0}) -> case emqx_rule_registry:find_resource_type(Type) of - {ok, #resource_type{}} -> - ResId = maps:get(id, Params, resource_id()), + {ok, #resource_type{on_create = {ModC, Create}, + on_destroy = {ModD, Destroy}, + params_spec = ParamSpec}} -> + Config = emqx_rule_validator:validate_params(Config0, ParamSpec), + ResId = resource_id(), try - _ = create_resource(maps:put(id, ResId, Params)), - true = is_source_alive(ResId), + _ = ?CLUSTER_CALL(init_resource, [ModC, Create, ResId, Config]), + _ = ?CLUSTER_CALL(clear_resource, [ModD, Destroy, ResId]), ok - catch E:R:S -> - ?LOG(warning, "test resource failed, ~0p:~0p ~0p", [E, R, S]), - {error, R} - after - _ = ?CLUSTER_CALL(delete_resource, [ResId]) + catch + throw:Reason -> {error, Reason} end; not_found -> {error, {resource_type_not_found, Type}} end. -is_source_alive(ResId) -> - case rpc:multicall(ekka_mnesia:running_nodes(), ?MODULE, get_resource_status, [ResId], 5000) of - {ResL, []} -> - is_source_alive_(ResL); - {_, _Errors} -> - false - end. - -is_source_alive_([]) -> true; -is_source_alive_([{ok, #{is_alive := true}} | ResL]) -> is_source_alive_(ResL); -is_source_alive_([{ok, #{is_alive := false}} | _ResL]) -> false; -is_source_alive_([_Error | _ResL]) -> false. - -spec(get_resource_status(resource_id()) -> {ok, resource_status()} | {error, Reason :: term()}). get_resource_status(ResId) -> case emqx_rule_registry:find_resource_params(ResId) of diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 313591ff3..39ac1e9c2 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -296,7 +296,7 @@ do_create_resource(Create, ParsedParams) -> list_resources(#{}, _Params) -> Data0 = lists:foldr(fun maybe_record_to_map/2, [], emqx_rule_registry:get_resources()), Data = lists:map(fun(Res = #{id := ResId}) -> - Status = emqx_rule_engine:is_source_alive(ResId), + Status = get_aggregated_status(ResId), maps:put(status, Status, Res) end, Data0), return({ok, Data}). @@ -304,6 +304,14 @@ list_resources(#{}, _Params) -> list_resources_by_type(#{type := Type}, _Params) -> return_all(emqx_rule_registry:get_resources_by_type(Type)). +get_aggregated_status(ResId) -> + lists:all(fun(Node) -> + case rpc:call(Node, emqx_rule_engine, get_resource_status, [ResId]) of + {ok, #{is_alive := true}} -> true; + _ -> false + end + end, ekka_mnesia:running_nodes()). + show_resource(#{id := Id}, _Params) -> case emqx_rule_registry:find_resource(Id) of {ok, R} -> From 16b8fc6edc162efa0a8c568f7e0f6866020736fe Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 29 Mar 2022 15:16:57 +0800 Subject: [PATCH 05/15] fix: emqx.appup.src --- src/emqx.appup.src | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 138bd495c..29c060b78 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,24 +1,12 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, -<<<<<<< HEAD [{"4.3.13", [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, -======= - [{"4.3.14", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, - {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]} - ]}, - {"4.3.13", - [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, ->>>>>>> 3b81db7ae (chore: export test function for ee's hot-conf) {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -33,7 +21,6 @@ {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -462,22 +449,12 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], -<<<<<<< HEAD - [{"4.3.13", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, -======= - [{"4.3.14", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, - {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]} - ]}, - {"4.3.13", + [{"4.3.13", [{load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, ->>>>>>> 3b81db7ae (chore: export test function for ee's hot-conf) {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, From cead8c905688f31c01a4433a245d586483d43768 Mon Sep 17 00:00:00 2001 From: EMQ-YangM Date: Wed, 30 Mar 2022 09:30:51 +0800 Subject: [PATCH 06/15] fix: use masp:get/3 to avoid crash after match failure --- apps/emqx_rule_engine/src/emqx_rule_events.erl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index fb7649e97..f3044ff74 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -172,18 +172,18 @@ eventmsg_connected(_ClientInfo = #{ is_bridge := IsBridge, mountpoint := Mountpoint }, - _ConnInfo = #{ + ConnInfo = #{ peername := PeerName, sockname := SockName, clean_start := CleanStart, proto_name := ProtoName, proto_ver := ProtoVer, - keepalive := Keepalive, connected_at := ConnectedAt, - conn_props := ConnProps, - receive_maximum := RcvMax, - expiry_interval := ExpiryInterval + receive_maximum := RcvMax }) -> + Keepalive = maps:get(keepalive, ConnInfo, 0), + ConnProps = maps:get(conn_props, ConnInfo, #{}), + ExpiryInterval = maps:get(expiry_interval, ConnInfo, 0), with_basic_columns('client.connected', #{clientid => ClientId, username => Username, From bf68735664acafec3420d81e8486984f1f6f85bd Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 30 Mar 2022 10:42:56 +0800 Subject: [PATCH 07/15] fix(frame): forbidden empty topic in strict mode --- src/emqx_frame.erl | 12 ++++++++++-- test/emqx_frame_SUITE.erl | 8 ++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/emqx_frame.erl b/src/emqx_frame.erl index 48e8b71fb..aa71b16ae 100644 --- a/src/emqx_frame.erl +++ b/src/emqx_frame.erl @@ -265,7 +265,7 @@ parse_packet(#mqtt_packet_header{type = ?CONNACK}, < - {TopicName, Rest} = parse_utf8_string(Bin, StrictMode), + {TopicName, Rest} = parse_topic_name(Bin, StrictMode), {PacketId, Rest1} = case QoS of ?QOS_0 -> {undefined, Rest}; _ -> parse_packet_id(Rest) @@ -357,7 +357,7 @@ parse_will_message(Packet = #mqtt_packet_connect{will_flag = true, proto_ver = Ver}, Bin, StrictMode) -> {Props, Rest} = parse_properties(Bin, Ver, StrictMode), - {Topic, Rest1} = parse_utf8_string(Rest, StrictMode), + {Topic, Rest1} = parse_topic_name(Rest, StrictMode), {Payload, Rest2} = parse_binary_data(Rest1), {Packet#mqtt_packet_connect{will_props = Props, will_topic = Topic, @@ -524,6 +524,14 @@ parse_binary_data(Bin) when 2 > byte_size(Bin) -> error(malformed_binary_data_length). +parse_topic_name(Bin, false) -> + parse_utf8_string(Bin, false); +parse_topic_name(Bin, true) -> + case parse_utf8_string(Bin, true) of + {<<>>, _Rest} -> error(empty_topic_name); + Result -> Result + end. + %%-------------------------------------------------------------------- %% Serialize MQTT Packet %%-------------------------------------------------------------------- diff --git a/test/emqx_frame_SUITE.erl b/test/emqx_frame_SUITE.erl index d98786e99..81c861bdb 100644 --- a/test/emqx_frame_SUITE.erl +++ b/test/emqx_frame_SUITE.erl @@ -162,6 +162,14 @@ t_parse_malformed_utf8_string(_) -> ParseState = emqx_frame:initial_parse_state(#{strict_mode => true}), ?catch_error(utf8_string_invalid, emqx_frame:parse(MalformedPacket, ParseState)). +t_parse_empty_topic_name(_) -> + Packet = <<48, 4, 0, 0, 0, 1>>, + NormalState = emqx_frame:initial_parse_state(#{strict_mode => false}), + ?assertMatch({_, _}, emqx_frame:parse(Packet, NormalState)), + + StrictState = emqx_frame:initial_parse_state(#{strict_mode => true}), + ?catch_error(empty_topic_name, emqx_frame:parse(Packet, StrictState)). + t_parse_frame_proxy_protocol(_) -> BinList = [ <<"PROXY TCP4 ">>, <<"PROXY TCP6 ">>, <<"PROXY UNKNOWN">> , <<"\r\n\r\n\0\r\nQUIT\n">>], From 22beba8ebdeca73a947304ffd1c9bbfc141fade1 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 30 Mar 2022 11:12:33 +0800 Subject: [PATCH 08/15] chore: update changes-4.3.md --- CHANGES-4.3.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 501ea34c4..ac7061057 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -17,6 +17,10 @@ File format: * In order to fix the execution order of exhook, e.g. before/after other plugins/modules, ExHook now supports user customizing emqx_hook execute priority. +### Bug fixes + +* Forbidden empty topics in strict mode + ## v4.3.13 ### Important changes From ea8ddcef60f71712164467ec9d8dddcef5f18d9c Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 31 Mar 2022 10:19:10 +0800 Subject: [PATCH 09/15] fix: fix spell error in CHANGES-4.3.md --- CHANGES-4.3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index ac7061057..a27122e59 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -19,7 +19,7 @@ File format: ### Bug fixes -* Forbidden empty topics in strict mode +* Prohibit empty topics in strict mode ## v4.3.13 From 510e4d31e4e941dfc00aa6029bb6b9aa12268ab7 Mon Sep 17 00:00:00 2001 From: EMQ-YangM Date: Thu, 31 Mar 2022 13:48:12 +0800 Subject: [PATCH 10/15] fix: load_module emqx_rule_events --- apps/emqx_rule_engine/src/emqx_rule_engine.appup.src | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 0ed0fee3a..e33341b4a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,7 +2,8 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.7", - [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -79,7 +80,8 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.7", - [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, From fe6af95f888b3703febd681691aea72927b659ce Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 31 Mar 2022 15:58:11 +0800 Subject: [PATCH 11/15] chore: remove useless stacktrace from catch. --- apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src | 2 +- apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 4 ++-- apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl | 2 +- apps/emqx_exhook/src/emqx_exhook_server.erl | 2 +- src/emqx_limiter.erl | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src index 8db4ffe84..7ec03e92d 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_jwt, [{description, "EMQ X Authentication with JWT"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.3.2"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_jwt_sup]}, {applications, [kernel,stdlib,jose]}, diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index b9831bb6f..1332ed53f 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,13 +1,13 @@ %% -*-: erlang -*- {VSN, [ - {"4.3.0", [ + {"4.3.[0-1]", [ {load_module, emqx_auth_jwt_svr, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ], [ - {"4.3.0", [ + {"4.3.[0-1]", [ {load_module, emqx_auth_jwt_svr, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl index b9d19bf57..f34cde783 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl @@ -91,7 +91,7 @@ do_init_jwks(Options) -> [K, V, Reason]), undefined; J -> J - catch T:R:_ -> + catch T:R -> ?LOG(warning, "Build ~p JWK ~p failed: {~p, ~p}~n", [K, V, T, R]), undefined diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index c4be91d07..d3953ade7 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -175,7 +175,7 @@ resovle_hookspec(HookSpecs) when is_list(HookSpecs) -> case maps:get(name, HookSpec, undefined) of undefined -> Acc; Name0 -> - Name = try binary_to_existing_atom(Name0, utf8) catch T:R:_ -> {T,R} end, + Name = try binary_to_existing_atom(Name0, utf8) catch T:R -> {T,R} end, case lists:member(Name, AvailableHooks) of true -> case lists:member(Name, MessageHooks) of diff --git a/src/emqx_limiter.erl b/src/emqx_limiter.erl index 1cca7140b..9a13e78f4 100644 --- a/src/emqx_limiter.erl +++ b/src/emqx_limiter.erl @@ -159,7 +159,7 @@ update_overall_limiter(Zone, Capacity, Interval) -> try esockd_limiter:update({Zone, overall_messages_routing}, Capacity, Interval), true - catch _:_:_ -> + catch _:_ -> false end. @@ -167,6 +167,6 @@ delete_overall_limiter(Zone) -> try esockd_limiter:delete({Zone, overall_messages_routing}), true - catch _:_:_ -> + catch _:_ -> false end. From 38447ff187b33760606f2d2c7a626a508000dd3d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 31 Mar 2022 10:10:12 +0200 Subject: [PATCH 12/15] chore(appup): fix update-appup.sh git reset before clean --- scripts/update-appup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh index d5f3b5407..d62faee74 100755 --- a/scripts/update-appup.sh +++ b/scripts/update-appup.sh @@ -93,6 +93,7 @@ if [ "$NEW_COPY" = 'no' ]; then REMOTE="$(git remote -v | grep "${GIT_REPO}" | head -1 | awk '{print $1}')" git fetch "$REMOTE" fi +git reset --hard git clean -fdx git checkout "${PREV_TAG}" make "$PROFILE" From 9cd63c87fc4e9f6e0293bcd49944c9cd5ba04aa2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 31 Mar 2022 10:25:57 +0200 Subject: [PATCH 13/15] docs: Update CHANGES-4.3.md to cover changes made in build env --- CHANGES-4.3.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 146f29e23..fe2988c77 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -16,6 +16,11 @@ File format: * For docker image, /opt/emqx/etc has been removed from the VOLUME list, this made it easier for the users to rebuild image on top with changed configs. +* CentOS 7 Erlang runtime is rebuilt on OpenSSL-1.1.1n (previously on 1.0), + Prior to v4.3.13, EMQX pick certain cipher suites proposed by the clients, + but then fail to handshake resulting in a `malformed_handshake_data` exception. +* CentOS 8 Erlang runtime is rebuilt on RockyLinux 8. + 'centos8' will remain in the package name to keep it backward compatible. ### Enhancements From b9c02bff1e02712a85350fc279d4fb6e195715ca Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 31 Mar 2022 10:39:30 +0200 Subject: [PATCH 14/15] ci: delete yum install openssl11 in test --- .ci/build_packages/tests.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 01fdc07dc..b74d61ff2 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -89,11 +89,6 @@ emqx_test(){ "rpm") packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm) - if [[ "${ARCH}" == "amd64" && $(rpm -E '%{rhel}') == 7 ]] ; then - # EMQX OTP requires openssl11 to have TLS1.3 support - yum install -y openssl11 - fi - rpm -ivh "${PACKAGE_PATH}/${packagename}" if ! rpm -q emqx | grep -q emqx; then echo "package install error" From c037c2cf6cb081f9118d1947d31b334db732cae2 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 31 Mar 2022 17:54:20 +0800 Subject: [PATCH 15/15] chore: release 4.3.13-rc.3 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 682072ae9..60501474d 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.13-rc.2"}). +-define(EMQX_RELEASE, {opensource, "4.3.13-rc.3"}). -else.