diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index 7f571ee70..d6b63180c 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -144,7 +144,17 @@ authorize_appid(_Method, _Path, Req) -> end. -ifdef(EMQX_ENTERPRISE). -filter(_) -> +filter(#{module := Module} = Route) -> + %% true if anything goes wrong + try + case erlang:function_exported(Module, filter, 1) of + true -> apply(Module, filter, [Route]); + false -> true + end + catch _:_ -> + true + end; +filter(_Route) -> true. -else. filter(#{app := emqx_modules}) -> true; @@ -156,7 +166,6 @@ filter(#{app := App}) -> end. -endif. - format(Port) when is_integer(Port) -> io_lib:format("0.0.0.0:~w", [Port]); format({Addr, Port}) when is_list(Addr) -> diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 5f206a155..40cdcd11d 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQX Retainer"}, - {vsn, "4.4.6"}, % strict semver, bump manually! + {vsn, "4.4.7"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_retainer/src/emqx_retainer.appup.src b/apps/emqx_retainer/src/emqx_retainer.appup.src index 3609fa78a..a41678b21 100644 --- a/apps/emqx_retainer/src/emqx_retainer.appup.src +++ b/apps/emqx_retainer/src/emqx_retainer.appup.src @@ -1,25 +1,21 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4\\.4\\.[1-5]">>, + [{<<"4\\.4\\.[1-6]">>, [{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}, {apply,{emqx_retainer_sup,ensure_worker_pool_started,[]}}, - {load_module,emqx_retainer,brutal_purge,soft_purge,[]} - ]}, + {load_module,emqx_retainer,brutal_purge,soft_purge,[]}]}, {"4.4.0", [{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}, {apply,{emqx_retainer_sup,ensure_worker_pool_started,[]}}, {load_module,emqx_retainer,brutal_purge,soft_purge,[]}, - {load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]} - ]}, + {load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{<<"4\\.4\\.[1-5]">>, + [{<<"4\\.4\\.[1-6]">>, [{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}, - {load_module,emqx_retainer,brutal_purge,soft_purge,[]} - ]}, + {load_module,emqx_retainer,brutal_purge,soft_purge,[]}]}, {"4.4.0", [{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_retainer,brutal_purge,soft_purge,[]}, - {load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]} - ]}, + {load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 281aacb6b..e1bf32d99 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -36,7 +36,7 @@ , on_message_publish/2 ]). --export([clean/1]). +-export([clean/1, read_messages/1]). %% for emqx_pool task func -export([dispatch/2]). diff --git a/apps/emqx_retainer/src/emqx_retainer_sup.erl b/apps/emqx_retainer/src/emqx_retainer_sup.erl index 02ced41ef..db08f2110 100644 --- a/apps/emqx_retainer/src/emqx_retainer_sup.erl +++ b/apps/emqx_retainer/src/emqx_retainer_sup.erl @@ -30,9 +30,15 @@ start_link(Env) -> supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]). +-dialyzer({no_match, [ensure_worker_pool_started/0]}). ensure_worker_pool_started() -> try - supervisor:start_child(?MODULE, worker_pool_spec()) + case is_managed_by_modules() of + true -> + supervisor:start_child(emqx_modules_sup, worker_pool_spec()); + false -> + supervisor:start_child(?MODULE, worker_pool_spec()) + end catch _:_ -> ignore end. diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl index 1507839c1..f6dd8215b 100644 --- a/apps/emqx_rule_engine/include/rule_engine.hrl +++ b/apps/emqx_rule_engine/include/rule_engine.hrl @@ -14,6 +14,8 @@ %% limitations under the License. %%-------------------------------------------------------------------- +-include_lib("emqx/include/logger.hrl"). + -define(APP, emqx_rule_engine). -define(KV_TAB, '@rule_engine_db'). @@ -187,11 +189,11 @@ case lists:filter(fun(ResParttern) -> false; (_) -> true end, ResL) of [] -> ResL; ErrL -> - ?LOG(error, "cluster_call error found, ResL: ~p", [ResL]), + ?LOG_SENSITIVE(error, "cluster_call error found, ResL: ~p", [ResL]), throw({Func, ErrL}) end; {ResL, BadNodes} -> - ?LOG(error, "cluster_call bad nodes found: ~p, ResL: ~p", [BadNodes, ResL]), + ?LOG_SENSITIVE(error, "cluster_call bad nodes found: ~p, ResL: ~p", [BadNodes, ResL]), throw({Func, {failed_on_nodes, BadNodes}}) end end()). diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index 4351a5410..d51ee61b6 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -20,7 +20,6 @@ -include("rule_engine.hrl"). -include("rule_actions.hrl"). -include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). -define(BAD_TOPIC_WITH_WILDCARD, wildcard_topic_not_allowed_for_publish). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 84b1de762..e679244b2 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -3,7 +3,7 @@ {vsn, "4.4.19"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry, emqx_rule_engine_jwt_sup]}, - {applications, [kernel,stdlib,rulesql,getopt,jose]}, + {applications, [kernel,stdlib,rulesql,getopt,jose,uuid]}, {mod, {emqx_rule_engine_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index c3b0154a1..6cd923618 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,25 +1,88 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.18",[{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}]}, + [{"4.4.18", + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt_worker,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}]}, {"4.4.17", - [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt_worker,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}]}, {"4.4.16", - [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt_worker,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.15", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt_worker,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {<<"4\\.4\\.1[3-4]">>, - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt_worker,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, @@ -27,7 +90,15 @@ {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.4.12", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, @@ -38,7 +109,13 @@ {update,emqx_rule_engine_jwt_sup,supervisor}, {load_module,emqx_rule_engine_jwt,brutal_purge,soft_purge,[]}]}, {"4.4.11", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_jwt_worker,brutal_purge,soft_purge,[]}, @@ -52,7 +129,11 @@ {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.4.10", - [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {add_module,emqx_rule_engine_jwt}, {add_module,emqx_rule_engine_jwt_worker}, {add_module,emqx_rule_engine_jwt_sup}, @@ -70,7 +151,10 @@ {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.9", - [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {add_module,emqx_rule_engine_jwt}, {add_module,emqx_rule_engine_jwt_worker}, {add_module,emqx_rule_engine_jwt_sup}, @@ -89,7 +173,10 @@ {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.4.8", - [{add_module,emqx_rule_engine_jwt}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_engine_jwt}, {add_module,emqx_rule_engine_jwt_worker}, {add_module,emqx_rule_engine_jwt_sup}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -108,7 +195,10 @@ {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {<<"4\\.4\\.[6-7]">>, - [{add_module,emqx_rule_engine_jwt}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_engine_jwt}, {add_module,emqx_rule_engine_jwt_worker}, {add_module,emqx_rule_engine_jwt_sup}, {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, @@ -127,7 +217,9 @@ {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.5", - [{add_module,emqx_rule_engine_jwt}, + [{load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_engine_jwt}, {add_module,emqx_rule_engine_jwt_worker}, {add_module,emqx_rule_engine_jwt_sup}, {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, @@ -147,7 +239,9 @@ {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}]}, {"4.4.4", - [{add_module,emqx_rule_engine_jwt}, + [{load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_engine_jwt}, {add_module,emqx_rule_engine_jwt_worker}, {add_module,emqx_rule_engine_jwt_sup}, {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, @@ -167,7 +261,9 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]}, {"4.4.3", - [{add_module,emqx_rule_engine_jwt}, + [{load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_engine_jwt}, {add_module,emqx_rule_engine_jwt_worker}, {add_module,emqx_rule_engine_jwt_sup}, {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, @@ -189,7 +285,8 @@ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.4.2", - [{add_module,emqx_rule_engine_jwt}, + [{load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_engine_jwt}, {add_module,emqx_rule_engine_jwt_worker}, {add_module,emqx_rule_engine_jwt_sup}, {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, @@ -212,7 +309,8 @@ {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.4.1", - [{add_module,emqx_rule_engine_jwt}, + [{load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_engine_jwt}, {add_module,emqx_rule_engine_jwt_worker}, {add_module,emqx_rule_engine_jwt_sup}, {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, @@ -235,7 +333,8 @@ {add_module,emqx_rule_date}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{add_module,emqx_rule_engine_jwt}, + [{load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_engine_jwt}, {add_module,emqx_rule_engine_jwt_worker}, {add_module,emqx_rule_engine_jwt_sup}, {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, @@ -258,25 +357,88 @@ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.18",[{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}]}, + [{"4.4.18", + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt_worker,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}]}, {"4.4.17", - [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt_worker,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}]}, {"4.4.16", - [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt_worker,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.15", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt_worker,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {<<"4\\.4\\.1[3-4]">>, - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt_worker,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, @@ -284,7 +446,15 @@ {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.4.12", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, @@ -295,7 +465,13 @@ {update,emqx_rule_engine_jwt_sup,supervisor}, {load_module,emqx_rule_engine_jwt,brutal_purge,soft_purge,[]}]}, {"4.4.11", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_jwt_worker,brutal_purge,soft_purge,[]}, @@ -309,7 +485,11 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.4.10", - [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -328,7 +508,10 @@ {delete_module,emqx_rule_engine_jwt_worker}, {delete_module,emqx_rule_engine_jwt}]}, {"4.4.9", - [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -348,7 +531,10 @@ {delete_module,emqx_rule_engine_jwt_worker}, {delete_module,emqx_rule_engine_jwt}]}, {"4.4.8", - [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, @@ -368,7 +554,10 @@ {delete_module,emqx_rule_engine_jwt_worker}, {delete_module,emqx_rule_engine_jwt}]}, {<<"4\\.4\\.[6-7]">>, - [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, @@ -388,7 +577,9 @@ {delete_module,emqx_rule_engine_jwt_worker}, {delete_module,emqx_rule_engine_jwt}]}, {"4.4.5", - [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, @@ -409,7 +600,9 @@ {delete_module,emqx_rule_engine_jwt_worker}, {delete_module,emqx_rule_engine_jwt}]}, {"4.4.4", - [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, @@ -430,7 +623,9 @@ {delete_module,emqx_rule_engine_jwt_worker}, {delete_module,emqx_rule_engine_jwt}]}, {"4.4.3", - [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, @@ -453,7 +648,8 @@ {delete_module,emqx_rule_engine_jwt_worker}, {delete_module,emqx_rule_engine_jwt}]}, {"4.4.2", - [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, @@ -477,7 +673,8 @@ {delete_module,emqx_rule_engine_jwt_worker}, {delete_module,emqx_rule_engine_jwt}]}, {"4.4.1", - [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, @@ -501,7 +698,8 @@ {delete_module,emqx_rule_engine_jwt_worker}, {delete_module,emqx_rule_engine_jwt}]}, {"4.4.0", - [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqlparser,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 03ae80bd9..9571ae221 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -17,7 +17,6 @@ -module(emqx_rule_engine). -include("rule_engine.hrl"). --include_lib("emqx/include/logger.hrl"). -export([ load_providers/0 , unload_providers/0 diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 951f9b4f2..2c0dba552 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -19,7 +19,6 @@ -behaviour(gen_server). -include("rule_engine.hrl"). --include_lib("emqx/include/logger.hrl"). -logger_header("[RuleEngineAPI]"). @@ -329,7 +328,7 @@ do_create_rule(Params) -> error -> do_create_rule2(ParsedParams) end; {error, Reason} -> - ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), + ?LOG_SENSITIVE(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), return({error, 400, ?ERR_BADARGS(Reason)}) end. @@ -339,7 +338,7 @@ do_create_rule2(ParsedParams) -> {error, {action_not_found, ActionName}} -> return({error, 400, ?ERR_NO_ACTION(ActionName)}); {error, Reason} -> - ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), + ?LOG_SENSITIVE(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), return({error, 400, ?ERR_BADARGS(Reason)}) end. @@ -352,11 +351,11 @@ delegate_update_rule(#{id := Id0}, Params) -> {error, {not_found, RuleId}} -> return({error, 400, ?ERR_NO_RULE(RuleId)}); {error, Reason} -> - ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), + ?LOG_SENSITIVE(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), return({error, 400, ?ERR_BADARGS(Reason)}) end; {error, Reason} -> - ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), + ?LOG_SENSITIVE(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), return({error, 400, ?ERR_BADARGS(Reason)}) end. @@ -409,7 +408,7 @@ delegate_create_resource(#{}, Params) -> fun() -> do_create_resource(create_resource, ParsedParams) end, Params); {error, Reason} -> - ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), + ?LOG_SENSITIVE(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), return({error, 400, ?ERR_BADARGS(Reason)}) end. @@ -434,7 +433,7 @@ do_create_resource2(Create, ParsedParams) -> {error, {init_resource, _}} -> return({error, 500, <<"Init resource failure!">>}); {error, Reason} -> - ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), + ?LOG_SENSITIVE(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), return({error, 400, ?ERR_BADARGS(Reason)}) end. @@ -483,7 +482,7 @@ delegate_start_resource(#{id := Id0}, _Params) -> {error, {resource_not_found, ResId}} -> return({error, 400, ?ERR_NO_RESOURCE(ResId)}); {error, Reason} -> - ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), + ?LOG_SENSITIVE(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), return({error, 400, ?ERR_BADARGS(Reason)}) end. @@ -508,7 +507,7 @@ delegate_update_resource(#{id := Id0}, NewParams) -> {error, {dependent_rules_exists, RuleIds}} -> return({error, 400, ?ERR_DEP_RULES_EXISTS(RuleIds)}); {error, Reason} -> - ?LOG(error, "Resource update failed: ~0p", [Reason]), + ?LOG_SENSITIVE(error, "Resource update failed: ~0p", [Reason]), return({error, 400, ?ERR_BADARGS(Reason)}) end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index f5c18d639..b15419ded 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -16,11 +16,10 @@ -module(emqx_rule_events). +-logger_header("[RuleEvents]"). + -include("rule_engine.hrl"). -include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). - --logger_header("[RuleEvents]"). -export([ load/1 , unload/0 diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index d308ff521..0531af225 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -216,6 +216,12 @@ , mongo_date/2 ]). +%% Random Funcs +-export([ random/0 + , uuid_v4/0 + , uuid_v4_no_hyphen/0 + ]). + %% Proc Dict Func -export([ proc_dict_get/1 , proc_dict_put/2 @@ -887,6 +893,18 @@ term_encode(Term) -> term_decode(Data) when is_binary(Data) -> erlang:binary_to_term(Data). +%%------------------------------------------------------------------------------ +%% Random Funcs +%%------------------------------------------------------------------------------ +random() -> + rand:uniform(). + +uuid_v4() -> + uuid_str(uuid:get_v4(), binary_standard). + +uuid_v4_no_hyphen() -> + uuid_str(uuid:get_v4(), binary_nodash). + %%------------------------------------------------------------------------------ %% Dict Funcs %%------------------------------------------------------------------------------ @@ -1039,3 +1057,6 @@ function_literal(Fun, [FArg | Args]) when is_atom(Fun), is_list(Args) -> end, WithFirstArg, Args) ++ ")"; function_literal(Fun, Args) -> {invalid_func, {Fun, Args}}. + +uuid_str(UUID, DisplyOpt) -> + uuid:uuid_to_string(UUID, DisplyOpt). diff --git a/apps/emqx_rule_engine/src/emqx_rule_monitor.erl b/apps/emqx_rule_engine/src/emqx_rule_monitor.erl index 4e4d4ebe1..19fa2c56b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_monitor.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_monitor.erl @@ -18,9 +18,8 @@ -behavior(gen_server). --include("rule_engine.hrl"). --include_lib("emqx/include/logger.hrl"). -logger_header("[Rule Monitor]"). +-include("rule_engine.hrl"). -export([init/1, handle_call/3, diff --git a/apps/emqx_rule_engine/src/emqx_rule_registry.erl b/apps/emqx_rule_engine/src/emqx_rule_registry.erl index 37bca84aa..6185efdb9 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_registry.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_registry.erl @@ -18,10 +18,8 @@ -behaviour(gen_server). --include("rule_engine.hrl"). --include_lib("emqx/include/logger.hrl"). - -logger_header("[RuleRegistry]"). +-include("rule_engine.hrl"). -export([start_link/0]). @@ -77,6 +75,8 @@ -export([ update_rules_cache/0 , clear_rules_cache/0 + , get_rules_from_cache/0 + , update_rules_cache_locally/0 ]). %% for debug purposes @@ -467,16 +467,19 @@ delete_resource_type(Type) -> init([]) -> _TableId = ets:new(?KV_TAB, [named_table, set, public, {write_concurrency, true}, {read_concurrency, true}]), + ok = ensure_table_subscribed(), {ok, #{}}. handle_call({add_rules, Rules}, _From, State) -> trans(fun lists:foreach/2, [fun insert_rule/1, Rules]), - _ = ?CLUSTER_CALL(update_rules_cache, []), + %% the multicall is necessary, because the other nodes maybe running an older emqx version + %% so the table has not been subscribed + update_rules_cache_on_all_nodes(), {reply, ok, State}; handle_call({remove_rules, Rules}, _From, State) -> trans(fun lists:foreach/2, [fun delete_rule/1, Rules]), - _ = ?CLUSTER_CALL(update_rules_cache, []), + update_rules_cache_on_all_nodes(), {reply, ok, State}; handle_call(Req, _From, State) -> @@ -484,13 +487,25 @@ handle_call(Req, _From, State) -> {reply, ignored, State}. handle_cast(update_rules_cache, State) -> - _ = update_rules_cache(), + ok = ensure_table_subscribed(), + ok = update_rules_cache(), {noreply, State}; handle_cast(Msg, State) -> ?LOG(error, "unexpected cast ~p", [Msg]), {noreply, State}. +handle_info({mnesia_table_event, {write, _Tab, _NewRule, _OldRules, _Tid} = Event}, State) -> + ?LOG(debug, "mnesia_table_event: ~p~n", [Event]), + ok = update_rules_cache_locally(), + {noreply, State}; + +handle_info({mnesia_table_event, {Delete, _Tab, _What, _OldRules, _Tid} = Event}, State) + when Delete =:= delete; Delete =:= delete_object -> + ?LOG(debug, "mnesia_table_event: ~p~n", [Event]), + ok = update_rules_cache_locally(), + {noreply, State}; + handle_info(Info, State) -> ?LOG(error, "unexpected info ~p", [Info]), {noreply, State}. @@ -504,6 +519,20 @@ code_change(_OldVsn, State, _Extra) -> %%------------------------------------------------------------------------------ %% Private functions %%------------------------------------------------------------------------------ +update_rules_cache_on_all_nodes() -> + ok = update_rules_cache(), + case ekka_mnesia:running_nodes() -- [node()] of + [] -> ok; + OtherNodes -> + _ = rpc:multicall(OtherNodes, ?MODULE, update_rules_cache_locally, [], 5000), + ok + end. + +ensure_table_subscribed() -> + case mnesia:subscribe({table, ?RULE_TAB, detailed}) of + {error, {already_exists, _}} -> ok; + {ok, _} -> ok + end. get_all_records(Tab) -> %mnesia:dirty_match_object(Tab, mnesia:table_info(Tab, wild_pattern)). diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 56c64924f..44fa65c44 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -19,7 +19,6 @@ -include("rule_engine.hrl"). -include("rule_actions.hrl"). -include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). -export([ apply_rule/2 , apply_rules/2 diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index 6ee2c3209..241cea493 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -15,7 +15,6 @@ -module(emqx_rule_sqltester). -include("rule_engine.hrl"). --include_lib("emqx/include/logger.hrl"). -export([ test/1 ]). diff --git a/apps/emqx_rule_engine/src/emqx_rule_utils.erl b/apps/emqx_rule_engine/src/emqx_rule_utils.erl index 78d77ed58..ea376f1ef 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_utils.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_utils.erl @@ -17,7 +17,6 @@ -module(emqx_rule_utils). -include("rule_engine.hrl"). --include_lib("emqx/include/logger.hrl"). -export([ replace_var/2 ]). diff --git a/changes/v4.4.19-en.md b/changes/v4.4.19-en.md index 6b9452aa5..34cc77afb 100644 --- a/changes/v4.4.19-en.md +++ b/changes/v4.4.19-en.md @@ -21,14 +21,26 @@ 2023-04-20T18:10:17.205436+08:00 [error] [esockd_proxy_protocol] The listener 127.0.0.1:8883 is working in proxy protocol mode, but timed out while waiting for proxy_protocol header ``` -- Adds a new feature to enable partial certificate chain validation for TLS listeners[#10553](https://github.com/emqx/emqx/pull/10553). +- Adds a new feature to enable partial certificate chain validation for TLS listeners [#10553](https://github.com/emqx/emqx/pull/10553). -- Adds a new feature to enable client certificate extended key usage validation for TLS listeners[#10669](https://github.com/emqx/emqx/pull/10669). + For details please checkout the `listener.ssl.external.partial_chain` in the `zones.conf` config file. + +- Adds a new feature to enable client certificate extended key usage validation for TLS listeners [#10669](https://github.com/emqx/emqx/pull/10669). + + For details please checkout the `listener.ssl.external.verify_peer_ext_key_usage` in the `zones.conf` config file. - Added the `live_connections` field in the HTTP API `/api/v4/nodes` response [#10859](https://github.com/emqx/emqx/pull/10859). Previously, this interface had a `connections` field, which represented the number of active connections on the current node that had not expired. This means that even if the MQTT connection has been disconnected, as long as the client has a persistent session, it would still be counted in the `connections` field. The newly added `live_connections` field specifically counts the number of clients with MQTT connections that have not been disconnected. +- Added 3 random SQL functions to the rule engine [#11113](https://github.com/emqx/emqx/pull/11113). + + - random(): Generates a random number between 0 and 1 (0.0 =< X < 1.0). + - uuid_v4(): Generates a random UUID (version 4) string. + - uuid_v4_no_hyphen(): Generates a random UUID (version 4) string without hyphens. + +- Added numerical range validation (23-65535) for the `mqtt.max_clientid_len` configuration parameter [#11096](https://github.com/emqx/emqx/pull/11096). + ## Bug fixes - Fixed an issue where the rule engine was unable to access variables exported by `FOREACH` in the `DO` clause [#10620](https://github.com/emqx/emqx/pull/10620). @@ -43,6 +55,10 @@ `[{"elem": "a","date": "undefined"}]`. After the fix, the output of the SQL statement is: `[{"elem": "a","date": "2023-05-06"}]` +- Fixed the issue where the cache of rules failed to update in certain cases [#11072](https://github.com/emqx/emqx/pull/11072). + + Prior to the fix, after manually updating the rules, there could be instances where the cache update did not synchronize to certain nodes. This would result in inconsistent rule execution states across different nodes. + - Fixed an issue where the WebHook plugin failed to execute the `on_client_connack` hook [#10710](https://github.com/emqx/emqx/pull/10710). See https://github.com/emqx/emqx/issues/10628 for more details. diff --git a/changes/v4.4.19-zh.md b/changes/v4.4.19-zh.md index cf0294713..c6768b03f 100644 --- a/changes/v4.4.19-zh.md +++ b/changes/v4.4.19-zh.md @@ -21,14 +21,26 @@ 2023-04-20T18:10:17.205436+08:00 [error] [esockd_proxy_protocol] The listener 127.0.0.1:8883 is working in proxy protocol mode, but timed out while waiting for proxy_protocol header ``` -- 增加了一个新功能,为 TLS 监听器启用部分证书链验证 [#10553](https://github.com/emqx/emqx/pull/10553)。 +- 增加了一个新功能,用户可以在 TLS 监听器中启用“部分证书链验证”了 [#10553](https://github.com/emqx/emqx/pull/10553)。 -- 增加了一个新功能,为 TLS 监听器启用客户端证书扩展密钥使用验证 [#10669](https://github.com/emqx/emqx/pull/10669)。 + 详情请查看 `zones.conf` 配置文件中的 `listener.ssl.external.partial_chain` 配置项。 + +- 增加了一个新功能,用户可以在 TLS 监听器中启用“客户端证书扩展密钥用途验证”了 [#10669](https://github.com/emqx/emqx/pull/10669)。 + + 详情请查看 `zones.conf` 配置文件中的 `listener.ssl.external.verify_peer_ext_key_usage` 配置项。 - 在 HTTP API `/api/v4/nodes` 的返回中增加 `live_connections` 字段 [#10859](https://github.com/emqx/emqx/pull/10859)。 此前该接口中有一个 `connections` 字段,它代表当前节点上会话未过期的连接数量。这意味着即使 MQTT 连接已经断开,只要客户端保持了会话,它仍然会被统计在 `connections` 中。新增的 `live_connections` 字段则仅仅统计 MQTT 连接未断开的客户端数量。 +- 规则引擎新增了三个随机函数 [#11113](https://github.com/emqx/emqx/pull/11113)。 + + - random():生成 0 到 1 之间的随机数 (0.0 =< X < 1.0)。 + - uuid_v4():生成随机的 UUID (version4) 字符串。 + - uuid_v4_no_hyphen():生成随机的不带连词符的 UUID (version4) 字符串。 + +- 为 `mqtt.max_clientid_len` 配置项增加数值范围校验 (23-65535) [#11096](https://github.com/emqx/emqx/pull/11096)。 + ## 修复 - 修复规则引擎无法在 `DO` 子句中访问 `FOREACH` 导出的变量的问题 [#10620](https://github.com/emqx/emqx/pull/10620)。 @@ -43,6 +55,10 @@ `[{"elem": "a","date": "undefined"}]`。 修复后,SQL 的输出为:`[{"elem": "a","date": "2023-05-06"}]` +- 修复在某些情况下,规则的缓存没能更新的问题 [#11072](https://github.com/emqx/emqx/pull/11072)。 + + 修复前,手动更新规则之后,可能会出现缓存的更新没能同步到某些节点上的情况,这会导致规则在不同的节点上运行状态不一致。 + - 修复 WebHook 插件执行 `on_client_connack` 钩子失败的问题 [#10710](https://github.com/emqx/emqx/pull/10710)。 详见 https://github.com/emqx/emqx/issues/10628 diff --git a/etc/emqx.conf b/etc/emqx.conf index 9abb6b414..84d0864ee 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1673,6 +1673,41 @@ crl_cache_refresh_interval = 15m ## Value: Ciphers listener.ssl.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +## When EMQX verifies a client certificate during the x509 path validation +## process, it constructs a certificate chain that starts with the client +## certificate and ends with a trust anchor. +## By default, if the setting is set to `false`, the trust anchor is the +## rootCA, and the certificate chain must be complete. +## However, if the setting is set to `true` or `cacert_from_cacertfile`, +## the last certificate in the cacertfile will be used as the trust anchor +## certificate (such as an intermediate CA). This creates a partial chain +## in the path validation. +## Alternatively, if the setting is set to `two_cacerts_from_cacertfile`, +## one of the last two certificates in the cacertfile will be used as the +## trust anchor certificate, forming a partial chain. This option is +## particularly useful for CA certificate rotation. +## However, please note that it incurs some additional overhead, so it +## should only be used for certificate rotation purposes. +## +## Values: false | true | cacert_from_cacertfile | two_cacerts_from_cacertfile +## listener.ssl.external.partial_chain = false + +## For additional client certificate validation, the value defined here must present in the +## 'Extended Key Usage' of client certificate defined in +## [rfc5280](https://www.rfc-editor.org/rfc/rfc5280#section-4.2.1.12). +## +## Allowed values are +## - "clientAuth" +## - "serverAuth" +## - "codeSigning" +## - "emailProtection" +## - "timeStamping" +## - "ocspSigning" +## - raw OID, example: "OID:1.3.6.1.5.5.7.3.2" +## Comma-separated string is also supported for validating the subset of key usages. +## example, "serverAuth,OID:1.3.6.1.5.5.7.3.2" +## +## listener.ssl.external.verify_peer_ext_key_usage = "clientAuth" ## Ciphers for TLS PSK. ## Note that 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot @@ -2576,7 +2611,7 @@ broker.route_batch_clean = off ## are mostly published to topics with large number of levels. ## ## NOTE: This is a cluster-wide configuration. -## It rquires all nodes to be stopped before changing it. +## It requires all nodes to be stopped before changing it. ## ## Value: Enum ## - true: enable trie path compaction diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 9f54519d9..d63bfb67c 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.19-alpha.1"}). +-define(EMQX_RELEASE, {opensource, "4.4.19"}). -else. diff --git a/priv/emqx.schema b/priv/emqx.schema index 9e2487757..7aa0592f2 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -276,6 +276,9 @@ end}. {validator, "range:1-65535", "must be 1 to 65535", fun(X) -> X >= 1 andalso X =< 65535 end}. +{validator, "range:23-65535", "must be 23 to 65535", + fun(X) -> X >= 23 andalso X =< 65535 end}. + {validator, "range:1-9", "must be 1 to 9", fun(X) -> X >= 1 andalso X =< 9 end}. @@ -955,7 +958,8 @@ end}. %% @doc Set the Max ClientId Length Allowed. {mapping, "mqtt.max_clientid_len", "emqx.max_clientid_len", [ {default, 65535}, - {datatype, integer} + {datatype, integer}, + {validators, ["range:23-65535"]} ]}. %% @doc Set the Maximum topic levels. diff --git a/rebar.config b/rebar.config index 83ab8fbcf..916608dc3 100644 --- a/rebar.config +++ b/rebar.config @@ -52,7 +52,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.11"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.0.1"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.11"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.12"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.5"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} @@ -67,6 +67,7 @@ , {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}} , {epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} + , {uuid, {git, "https://github.com/okeuday/uuid.git", {tag, "v2.0.6"}}} ]}. {xref_ignores, diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index 632fed25c..0ba0d82ec 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -14,7 +14,7 @@ case "${PKG_VSN}" in 4.4*) # keep the above 4.3 untouched, otherwise conflicts! EMQX_CE_DASHBOARD_VERSION='v4.4.11' - EMQX_EE_DASHBOARD_VERSION='v4.4.24' + EMQX_EE_DASHBOARD_VERSION='v4.4.26' ;; *) echo "Unsupported version $PKG_VSN" >&2 @@ -49,6 +49,7 @@ if [ -d "$DASHBOARD_PATH/www" ] && [ "$(version)" = "$VERSION" ]; then exit 0 fi +echo "Downloading dashboard from $DIRECT_DOWNLOAD_URL" curl -L --silent --show-error \ --header "Accept: application/octet-stream" \ --output "${RELEASE_ASSET_FILE}" \ diff --git a/src/emqx.appup.src b/src/emqx.appup.src index cb76375f1..effdab40f 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,7 +2,9 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.18", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, @@ -13,7 +15,8 @@ {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.4.17", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, @@ -25,7 +28,8 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.4.16", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, @@ -40,7 +44,8 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.4.15", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, @@ -57,7 +62,8 @@ {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.4.14", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -79,7 +85,8 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions_trans,brutal_purge,soft_purge,[]}]}, {"4.4.13", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]}, @@ -101,7 +108,8 @@ {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.4.12", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]}, @@ -123,7 +131,8 @@ {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.4.11", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]}, @@ -638,7 +647,9 @@ [gen_rpc,insecure_auth_fallback_allowed,true]}}]}, {<<".*">>,[]}], [{"4.4.18", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, @@ -648,7 +659,8 @@ {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.4.17", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, @@ -659,7 +671,8 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.4.16", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, @@ -673,7 +686,8 @@ {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.4.15", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, @@ -689,7 +703,8 @@ {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.4.14", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -710,7 +725,8 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions_trans,brutal_purge,soft_purge,[]}]}, {"4.4.13", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]}, @@ -731,7 +747,8 @@ {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.4.12", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]}, @@ -752,7 +769,8 @@ {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.4.11", - [{load_module,emqx_zone,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_zone,brutal_purge,soft_purge,[]}, {load_module,emqx_cm_locker,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_hooks.erl b/src/emqx_hooks.erl index c7ec23fcf..aaf92a44a 100644 --- a/src/emqx_hooks.erl +++ b/src/emqx_hooks.erl @@ -21,6 +21,9 @@ -include("logger.hrl"). -include("types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -endif. @@ -285,6 +288,7 @@ handle_cast({del, HookPoint, Action}, State) -> Callbacks -> ok = insert_hook(HookPoint, Callbacks) end, + ?tp(debug, emqx_hook_removed, #{hookpoint => HookPoint, action => Action}), {noreply, State}; handle_cast(Msg, State) -> diff --git a/src/emqx_listeners.erl b/src/emqx_listeners.erl index baa840d04..13be71cf3 100644 --- a/src/emqx_listeners.erl +++ b/src/emqx_listeners.erl @@ -140,7 +140,8 @@ start_listener(Proto, ListenOn, Options0) when Proto == ssl; Proto == tls -> Options1 = proplists:delete(listener_id, Options0), Options2 = emqx_ocsp_cache:inject_sni_fun(ListenerID, Options1), Options3 = emqx_tls_lib:inject_root_fun(Options2), - Options = emqx_tls_lib:inject_verify_fun(Options3), + Options4 = emqx_tls_lib:inject_verify_fun(Options3), + Options = emqx_tls_lib:maybe_drop_incompatible_options(Options4), ok = maybe_register_crl_urls(Options), start_mqtt_listener('mqtt:ssl', ListenOn, Options); diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index 7f7d531c0..3ca36f26c 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -253,13 +253,15 @@ default_plugins() -> %% default is true in data/load_modules. **NOT HERE** {emqx_retainer, false}, {emqx_recon, false}, - %% emqx_telemetry is not exist in enterprise. + %% emqx_telemetry does not exist in enterprise. %% {emqx_telemetry, false}, {emqx_rule_engine, true}, {emqx_bridge_mqtt, false}, {emqx_schema_registry, true}, {emqx_eviction_agent, true}, - {emqx_node_rebalance, true} + {emqx_node_rebalance, true}, + %% emqx_gcp_device is managed by emqx_modules. + {emqx_gcp_device, false} ]. -endif. diff --git a/src/emqx_tls_lib.erl b/src/emqx_tls_lib.erl index 6aca651f5..9a05bd64d 100644 --- a/src/emqx_tls_lib.erl +++ b/src/emqx_tls_lib.erl @@ -26,6 +26,7 @@ , inject_verify_fun/1 , opt_partial_chain/1 , opt_verify_fun/1 + , maybe_drop_incompatible_options/1 ]). -include("logger.hrl"). @@ -223,15 +224,20 @@ replace(Opts, Key, Value) -> [{Key, Value} | proplists:delete(Key, Opts)]. %% @doc Helper, make TLS root_fun rootfun_trusted_ca_from_cacertfile(NumOfCerts, SslOpts) -> Cacertfile = proplists:get_value(cacertfile, SslOpts, undefined), - try do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, Cacertfile) - catch _Error:_Info:ST -> - %% The cacertfile will be checked by OTP SSL as well and OTP choice to be silent on this. - %% We are touching security sutffs, don't leak extra info.. - ?LOG(error, "Failed to look for trusted cacert from cacertfile. Stacktrace: ~p", [ST]), - throw({error, ?FUNCTION_NAME}) + case file:read_file(Cacertfile) of + {ok, PemBin} -> + try do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, PemBin) + catch _Error:_Info:ST -> + %% The cacertfile will be checked by OTP SSL as well and OTP choice to be silent on this. + %% We are touching security sutffs, don't leak extra info.. + ?LOG(error, "Failed to look for trusted cacert from cacertfile. Stacktrace: ~p", [ST]), + throw({error, ?FUNCTION_NAME}) + end; + {error, Reason} -> + throw({error, {read_cacertfile_error, Cacertfile, Reason}}) end. -do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, Cacertfile) -> - {ok, PemBin} = file:read_file(Cacertfile), + +do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, PemBin) -> %% The last one or two should be the top parent in the chain if it is a chain Certs = public_key:pem_decode(PemBin), Pos = length(Certs) - NumOfCerts + 1, @@ -239,6 +245,23 @@ do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, Cacertfile) -> lists:sublist(public_key:pem_decode(PemBin), Pos, NumOfCerts)], emqx_const_v2:make_tls_root_fun(cacert_from_cacertfile, Trusted). +maybe_drop_incompatible_options(Options) -> + case proplists:get_value(ssl_options, Options) of + undefined -> + Options; + SslOpts -> + maybe_drop_incompatible_options(Options, SslOpts, lists:keyfind(versions, 1, SslOpts)) + end. + +maybe_drop_incompatible_options(Options, _SslOpts, false) -> + Options; +maybe_drop_incompatible_options(Options, SslOpts0, {versions, ['tlsv1.3']}) -> + Incompatible = [reuse_sessions, secure_renegotiate, user_lookup_fun, client_renegotiation], + SslOpts = lists:filter(fun({K, _V}) -> not lists:member(K, Incompatible) end, SslOpts0), + lists:keyreplace(ssl_options, 1, Options, {ssl_options, SslOpts}); +maybe_drop_incompatible_options(Options, _SslOpts, {versions, [_ | _]}) -> + Options. + -if(?OTP_RELEASE > 22). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -261,5 +284,18 @@ drop_tls13_no_versions_cipers_test() -> has_tlsv13_cipher(Ciphers) -> lists:any(fun(C) -> lists:member(C, Ciphers) end, ?TLSV13_EXCLUSIVE_CIPHERS). +maybe_drop_incompatible_options_test() -> + Opts0 = [{ssl_options, [{versions, ['tlsv1.3']}, {ciphers, ?TLSV13_EXCLUSIVE_CIPHERS}, + {reuse_sessions, true}, {secure_renegotiate, true}, + {user_lookup_fun, fun maybe_drop_incompatible_options/1}, + {client_renegotiation, true}]}], + Opts = maybe_drop_incompatible_options(Opts0), + ?assertNot(lists:member(reuse_sessions, proplists:get_value(ssl_options, Opts))), + ?assertNot(lists:member(secure_renegotiate, proplists:get_value(ssl_options, Opts))), + ?assertNot(lists:member(user_lookup_fun, proplists:get_value(ssl_options, Opts))), + ?assertNot(lists:member(client_renegotiation, proplists:get_value(ssl_options, Opts))), + ?assertEqual([{versions, ['tlsv1.3']}, {ciphers, ?TLSV13_EXCLUSIVE_CIPHERS}], + proplists:get_value(ssl_options, Opts)). + -endif. %% TEST -endif. %% OTP_RELEASE > 22 diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl index 6e0994757..3d5a9712f 100644 --- a/test/emqx_plugins_SUITE.erl +++ b/test/emqx_plugins_SUITE.erl @@ -119,7 +119,8 @@ default_plugins() -> {emqx_rule_engine, true}, {emqx_schema_registry, true}, {emqx_eviction_agent, true}, - {emqx_node_rebalance, true} + {emqx_node_rebalance, true}, + {emqx_gcp_device, false} ].