From 08da5f526713e3cd596d608094cdefb0bbd01042 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 21 Dec 2021 16:53:06 +0800 Subject: [PATCH 01/99] fix(boot_script): LD_LIBRARY_PATH: unbound variable --- bin/emqx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index 7862a98b8..b5f7a1903 100755 --- a/bin/emqx +++ b/bin/emqx @@ -204,7 +204,10 @@ fi if ! check_erlang_start >/dev/null 2>&1; then BUILT_ON="$(head -1 "${REL_DIR}/BUILT_ON")" ## failed to start, might be due to missing libs, try to be portable - export LD_LIBRARY_PATH="$DYNLIBS_DIR:$LD_LIBRARY_PATH" + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-$DYNLIBS_DIR}" + if [ "$LD_LIBRARY_PATH" != "$DYNLIBS_DIR" ]; then + export LD_LIBRARY_PATH="$DYNLIBS_DIR:$LD_LIBRARY_PATH" + fi if ! check_erlang_start; then ## it's hopeless echoerr "FATAL: Unable to start Erlang." From 25b7719db5faf95330a5ee91df44fb473349b142 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 21 Dec 2021 17:00:49 +0800 Subject: [PATCH 02/99] feat(authz): support sync configuration in the cluster --- apps/emqx_authz/src/emqx_authz.erl | 30 ++++++++----------- .../src/emqx_authz_api_settings.erl | 7 +++-- apps/emqx_authz/src/emqx_authz_utils.erl | 12 ++++++-- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 54438793b..97e0fb595 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -33,7 +33,6 @@ , move/2 , move/3 , update/2 - , update/3 , authorize/5 ]). @@ -114,24 +113,18 @@ move(Type, Cmd) -> move(Type, Cmd, #{}). move(Type, #{<<"before">> := Before}, Opts) -> - emqx:update_config( ?CONF_KEY_PATH - , {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}, Opts); + emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}, Opts); move(Type, #{<<"after">> := After}, Opts) -> - emqx:update_config( ?CONF_KEY_PATH - , {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))}, Opts); + emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))}, Opts); move(Type, Position, Opts) -> - emqx:update_config( ?CONF_KEY_PATH - , {?CMD_MOVE, type(Type), Position}, Opts). + emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), Position}, Opts). +update({?CMD_REPLACE, Type}, Sources) -> + emqx_authz_utils:update_config(?CONF_KEY_PATH, {{?CMD_REPLACE, type(Type)}, Sources}); +update({?CMD_DELETE, Type}, Sources) -> + emqx_authz_utils:update_config(?CONF_KEY_PATH, {{?CMD_DELETE, type(Type)}, Sources}); update(Cmd, Sources) -> - update(Cmd, Sources, #{}). - -update({?CMD_REPLACE, Type}, Sources, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {{?CMD_REPLACE, type(Type)}, Sources}, Opts); -update({?CMD_DELETE, Type}, Sources, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {{?CMD_DELETE, type(Type)}, Sources}, Opts); -update(Cmd, Sources, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {Cmd, Sources}, Opts). + emqx_authz_utils:update_config(?CONF_KEY_PATH, {Cmd, Sources}). do_update({?CMD_MOVE, Type, ?CMD_MOVE_TOP}, Conf) when is_list(Conf) -> {Source, Front, Rear} = take(Type, Conf), @@ -155,8 +148,8 @@ do_update({?CMD_APPEND, Sources}, Conf) when is_list(Sources), is_list(Conf) -> NConf = Conf ++ Sources, ok = check_dup_types(NConf), NConf; -do_update({{?CMD_REPLACE, Type}, #{<<"enable">> := true} = Source}, Conf) when is_map(Source), - is_list(Conf) -> +do_update({{?CMD_REPLACE, Type}, #{<<"enable">> := true} = Source}, Conf) + when is_map(Source), is_list(Conf) -> case create_dry_run(Type, Source) of ok -> {_Old, Front, Rear} = take(Type, Conf), @@ -165,7 +158,8 @@ do_update({{?CMD_REPLACE, Type}, #{<<"enable">> := true} = Source}, Conf) when i NConf; {error, _} = Error -> Error end; -do_update({{?CMD_REPLACE, Type}, Source}, Conf) when is_map(Source), is_list(Conf) -> +do_update({{?CMD_REPLACE, Type}, Source}, Conf) + when is_map(Source), is_list(Conf) -> {_Old, Front, Rear} = take(Type, Conf), NConf = Front ++ [Source | Rear], ok = check_dup_types(NConf), diff --git a/apps/emqx_authz/src/emqx_authz_api_settings.erl b/apps/emqx_authz/src/emqx_authz_api_settings.erl index c2a87da16..c7d75bbba 100644 --- a/apps/emqx_authz/src/emqx_authz_api_settings.erl +++ b/apps/emqx_authz/src/emqx_authz_api_settings.erl @@ -54,8 +54,9 @@ settings(get, _Params) -> settings(put, #{body := #{<<"no_match">> := NoMatch, <<"deny_action">> := DenyAction, <<"cache">> := Cache}}) -> - {ok, _} = emqx:update_config([authorization, no_match], NoMatch), - {ok, _} = emqx:update_config([authorization, deny_action], DenyAction), - {ok, _} = emqx:update_config([authorization, cache], Cache), + {ok, _} = emqx_authz_utils:update_config([authorization, no_match], NoMatch), + {ok, _} = emqx_authz_utils:update_config( + [authorization, deny_action], DenyAction), + {ok, _} = emqx_authz_utils:update_config([authorization, cache], Cache), ok = emqx_authz_cache:drain_cache(), {200, authorization_settings()}. diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl index 73132aacb..78140d118 100644 --- a/apps/emqx_authz/src/emqx_authz_utils.erl +++ b/apps/emqx_authz/src/emqx_authz_utils.erl @@ -18,9 +18,11 @@ -include_lib("emqx/include/emqx_placeholder.hrl"). --export([cleanup_resources/0, - make_resource_id/1, - create_resource/2]). +-export([ cleanup_resources/0 + , make_resource_id/1 + , create_resource/2 + , update_config/2 + ]). -define(RESOURCE_GROUP, <<"emqx_authz">>). @@ -45,6 +47,10 @@ make_resource_id(Name) -> NameBin = bin(Name), emqx_resource:generate_id(?RESOURCE_GROUP, NameBin). +update_config(Path, ConfigRequest) -> + emqx_conf:update(Path, ConfigRequest, #{rawconf_with_defaults => true, + override_to => cluster}). + %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ From c3eab1618288fd64caa2990cdb487da31cefa3ae Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 21 Dec 2021 19:50:36 +0800 Subject: [PATCH 03/99] fix(boot_script): wait until the emqx running --- bin/emqx | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index b5f7a1903..00133ed0b 100755 --- a/bin/emqx +++ b/bin/emqx @@ -431,6 +431,26 @@ wait_for() { done } +wait_until_return_val() { + local RESULT + local WAIT_TIME + local CMD + RESULT="$1" + WAIT_TIME="$2" + shift 2 + CMD="$*" + while true; do + if [ "$($CMD 2>/dev/null)" = "$RESULT" ]; then + return 0 + fi + if [ "$WAIT_TIME" -le 0 ]; then + return 1 + fi + WAIT_TIME=$((WAIT_TIME - 1)) + sleep 1 + done +} + latest_vm_args() { local hint_var_name="$1" local vm_args_file @@ -556,7 +576,8 @@ case "${COMMAND}" in "$(relx_start_command)" WAIT_TIME=${WAIT_FOR_ERLANG:-15} - if wait_for "$WAIT_TIME" 'relx_nodetool' 'ping'; then + if wait_until_return_val "true" "$WAIT_TIME" 'relx_nodetool' \ + 'eval' 'emqx:is_running()'; then echo "$EMQX_DESCRIPTION $REL_VSN is started successfully!" exit 0 else From 80c1128b7a23f947b027ebca2fb076bd0011e071 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 23 Dec 2021 11:45:58 +0800 Subject: [PATCH 04/99] fix(authz): fix deadlock issues --- apps/emqx_authz/src/emqx_authz_http.erl | 4 ++-- apps/emqx_authz/src/emqx_authz_mongodb.erl | 4 ++-- apps/emqx_authz/src/emqx_authz_mysql.erl | 4 ++-- apps/emqx_authz/src/emqx_authz_postgresql.erl | 4 ++-- apps/emqx_authz/src/emqx_authz_redis.erl | 4 ++-- apps/emqx_authz/src/emqx_authz_utils.erl | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index c2ee96594..fe48b35d0 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -48,12 +48,12 @@ init(#{url := Url} = Source) -> end. destroy(#{annotations := #{id := Id}}) -> - ok = emqx_resource:remove(Id). + ok = emqx_resource:remove_local(Id). dry_run(Source) -> URIMap = maps:get(url, Source), NSource = maps:put(base_url, maps:remove(query, URIMap), Source), - emqx_resource:create_dry_run(emqx_connector_http, NSource). + emqx_resource:create_dry_run_local(emqx_connector_http, NSource). authorize(Client, PubSub, Topic, #{type := http, diff --git a/apps/emqx_authz/src/emqx_authz_mongodb.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl index 97fac9627..84ef2f302 100644 --- a/apps/emqx_authz/src/emqx_authz_mongodb.erl +++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl @@ -46,10 +46,10 @@ init(Source) -> end. dry_run(Source) -> - emqx_resource:create_dry_run(emqx_connector_mongo, Source). + emqx_resource:create_dry_run_local(emqx_connector_mongo, Source). destroy(#{annotations := #{id := Id}}) -> - ok = emqx_resource:remove(Id). + ok = emqx_resource:remove_local(Id). authorize(Client, PubSub, Topic, #{collection := Collection, diff --git a/apps/emqx_authz/src/emqx_authz_mysql.erl b/apps/emqx_authz/src/emqx_authz_mysql.erl index a5b14ec1b..181478a76 100644 --- a/apps/emqx_authz/src/emqx_authz_mysql.erl +++ b/apps/emqx_authz/src/emqx_authz_mysql.erl @@ -48,10 +48,10 @@ init(#{query := SQL} = Source) -> end. dry_run(Source) -> - emqx_resource:create_dry_run(emqx_connector_mysql, Source). + emqx_resource:create_dry_run_local(emqx_connector_mysql, Source). destroy(#{annotations := #{id := Id}}) -> - ok = emqx_resource:remove(Id). + ok = emqx_resource:remove_local(Id). authorize(Client, PubSub, Topic, #{annotations := #{id := ResourceID, diff --git a/apps/emqx_authz/src/emqx_authz_postgresql.erl b/apps/emqx_authz/src/emqx_authz_postgresql.erl index f101841c2..4e84fe418 100644 --- a/apps/emqx_authz/src/emqx_authz_postgresql.erl +++ b/apps/emqx_authz/src/emqx_authz_postgresql.erl @@ -48,10 +48,10 @@ init(#{query := SQL} = Source) -> end. destroy(#{annotations := #{id := Id}}) -> - ok = emqx_resource:remove(Id). + ok = emqx_resource:remove_local(Id). dry_run(Source) -> - emqx_resource:create_dry_run(emqx_connector_pgsql, Source). + emqx_resource:create_dry_run_local(emqx_connector_pgsql, Source). parse_query(Sql) -> case re:run(Sql, ?RE_PLACEHOLDER, [global, {capture, all, list}]) of diff --git a/apps/emqx_authz/src/emqx_authz_redis.erl b/apps/emqx_authz/src/emqx_authz_redis.erl index 1f6abe330..8765734cf 100644 --- a/apps/emqx_authz/src/emqx_authz_redis.erl +++ b/apps/emqx_authz/src/emqx_authz_redis.erl @@ -46,10 +46,10 @@ init(Source) -> end. destroy(#{annotations := #{id := Id}}) -> - ok = emqx_resource:remove(Id). + ok = emqx_resource:remove_local(Id). dry_run(Source) -> - emqx_resource:create_dry_run(emqx_connector_redis, Source). + emqx_resource:create_dry_run_local(emqx_connector_redis, Source). authorize(Client, PubSub, Topic, #{cmd := CMD, diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl index 78140d118..435388e44 100644 --- a/apps/emqx_authz/src/emqx_authz_utils.erl +++ b/apps/emqx_authz/src/emqx_authz_utils.erl @@ -32,7 +32,7 @@ create_resource(Module, Config) -> ResourceID = make_resource_id(Module), - case emqx_resource:create(ResourceID, Module, Config) of + case emqx_resource:create_local(ResourceID, Module, Config) of {ok, already_created} -> {ok, ResourceID}; {ok, _} -> {ok, ResourceID}; {error, Reason} -> {error, Reason} @@ -40,7 +40,7 @@ create_resource(Module, Config) -> cleanup_resources() -> lists:foreach( - fun emqx_resource:remove/1, + fun emqx_resource:remove_local/1, emqx_resource:list_group_instances(?RESOURCE_GROUP)). make_resource_id(Name) -> From 33a6568654cfb8629abb79094d18b22ece0e216b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 24 Dec 2021 10:00:00 +0800 Subject: [PATCH 05/99] fix(banned): crash by bad peerhost; add banned http API tests --- apps/emqx/src/emqx_banned.erl | 64 ++++---- apps/emqx/test/emqx_banned_SUITE.erl | 16 +- .../src/emqx_mgmt_api_banned.erl | 29 ++-- .../test/emqx_mgmt_banned_api_SUITE.erl | 144 ++++++++++++++++++ 4 files changed, 203 insertions(+), 50 deletions(-) create mode 100644 apps/emqx_management/test/emqx_mgmt_banned_api_SUITE.erl diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl index 53a71736a..091afae8a 100644 --- a/apps/emqx/src/emqx_banned.erl +++ b/apps/emqx/src/emqx_banned.erl @@ -37,7 +37,6 @@ , info/1 , format/1 , parse/1 - , to_timestamp/1 ]). %% gen_server callbacks @@ -53,6 +52,11 @@ -define(BANNED_TAB, ?MODULE). +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + %%-------------------------------------------------------------------- %% Mnesia bootstrap %%-------------------------------------------------------------------- @@ -106,32 +110,36 @@ format(#banned{who = Who0, }. parse(Params) -> - Who = pares_who(Params), - By = maps:get(<<"by">>, Params, <<"mgmt_api">>), - Reason = maps:get(<<"reason">>, Params, <<"">>), - At = parse_time(maps:get(<<"at">>, Params, undefined), erlang:system_time(second)), - Until = parse_time(maps:get(<<"until">>, Params, undefined), At + 5 * 60), - #banned{ - who = Who, - by = By, - reason = Reason, - at = At, - until = Until - }. - + case pares_who(Params) of + {error, Reason} -> {error, Reason}; + Who -> + By = maps:get(<<"by">>, Params, <<"mgmt_api">>), + Reason = maps:get(<<"reason">>, Params, <<"">>), + At = maps:get(<<"at">>, Params, erlang:system_time(second)), + Until = maps:get(<<"until">>, Params, At + 5 * 60), + case Until > erlang:system_time(second) of + true -> + #banned{ + who = Who, + by = By, + reason = Reason, + at = At, + until = Until + }; + false -> + {error, "already_expired"} + end + end. pares_who(#{as := As, who := Who}) -> pares_who(#{<<"as">> => As, <<"who">> => Who}); pares_who(#{<<"as">> := peerhost, <<"who">> := Peerhost0}) -> - {ok, Peerhost} = inet:parse_address(binary_to_list(Peerhost0)), - {peerhost, Peerhost}; + case inet:parse_address(binary_to_list(Peerhost0)) of + {ok, Peerhost} -> {peerhost, Peerhost}; + {error, einval} -> {error, "bad peerhost"} + end; pares_who(#{<<"as">> := As, <<"who">> := Who}) -> {As, Who}. -parse_time(undefined, Default) -> - Default; -parse_time(Rfc3339, _Default) -> - to_timestamp(Rfc3339). - maybe_format_host({peerhost, Host}) -> AddrBinary = list_to_binary(inet:ntoa(Host)), {peerhost, AddrBinary}; @@ -141,11 +149,6 @@ maybe_format_host({As, Who}) -> to_rfc3339(Timestamp) -> list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). -to_timestamp(Rfc3339) when is_binary(Rfc3339) -> - to_timestamp(binary_to_list(Rfc3339)); -to_timestamp(Rfc3339) -> - calendar:rfc3339_to_system_time(Rfc3339, [{unit, second}]). - -spec(create(emqx_types:banned() | map()) -> {ok, emqx_types:banned()} | {error, {already_exist, emqx_types:banned()}}). create(#{who := Who, @@ -168,10 +171,11 @@ create(Banned = #banned{who = Who}) -> mria:dirty_write(?BANNED_TAB, Banned), {ok, Banned}; [OldBanned = #banned{until = Until}] -> - case Until < erlang:system_time(second) of - true -> - {error, {already_exist, OldBanned}}; - false -> + %% Don't support shorten or extend the until time by overwrite. + %% We don't support update api yet, user must delete then create new one. + case Until > erlang:system_time(second) of + true -> {error, {already_exist, OldBanned}}; + false -> %% overwrite expired one is ok. mria:dirty_write(?BANNED_TAB, Banned), {ok, Banned} end diff --git a/apps/emqx/test/emqx_banned_SUITE.erl b/apps/emqx/test/emqx_banned_SUITE.erl index e09d0baae..4ba45c5ad 100644 --- a/apps/emqx/test/emqx_banned_SUITE.erl +++ b/apps/emqx/test/emqx_banned_SUITE.erl @@ -39,9 +39,13 @@ t_add_delete(_) -> by = <<"banned suite">>, reason = <<"test">>, at = erlang:system_time(second), - until = erlang:system_time(second) + 1000 + until = erlang:system_time(second) + 1 }, {ok, _} = emqx_banned:create(Banned), + {error, {already_exist, Banned}} = emqx_banned:create(Banned), + ?assertEqual(1, emqx_banned:info(size)), + {error, {already_exist, Banned}} = + emqx_banned:create(Banned#banned{until = erlang:system_time(second) + 100}), ?assertEqual(1, emqx_banned:info(size)), ok = emqx_banned:delete({clientid, <<"TestClient">>}), @@ -68,10 +72,14 @@ t_check(_) -> username => <<"user">>, peerhost => {127,0,0,1} }, + ClientInfo5 = #{}, + ClientInfo6 = #{clientid => <<"client1">>}, ?assert(emqx_banned:check(ClientInfo1)), ?assert(emqx_banned:check(ClientInfo2)), ?assert(emqx_banned:check(ClientInfo3)), ?assertNot(emqx_banned:check(ClientInfo4)), + ?assertNot(emqx_banned:check(ClientInfo5)), + ?assertNot(emqx_banned:check(ClientInfo6)), ok = emqx_banned:delete({clientid, <<"BannedClient">>}), ok = emqx_banned:delete({username, <<"BannedUser">>}), ok = emqx_banned:delete({peerhost, {192,168,0,1}}), @@ -83,8 +91,10 @@ t_check(_) -> t_unused(_) -> {ok, Banned} = emqx_banned:start_link(), - {ok, _} = emqx_banned:create(#banned{who = {clientid, <<"BannedClient">>}, - until = erlang:system_time(second)}), + {ok, _} = emqx_banned:create(#banned{who = {clientid, <<"BannedClient1">>}, + until = erlang:system_time(second)}), + {ok, _} = emqx_banned:create(#banned{who = {clientid, <<"BannedClient2">>}, + until = erlang:system_time(second) - 1}), ?assertEqual(ignored, gen_server:call(Banned, unexpected_req)), ?assertEqual(ok, gen_server:cast(Banned, unexpected_msg)), ?assertEqual(ok, Banned ! ok), diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 6521a549c..911b298a1 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -105,7 +105,7 @@ fields(ban) -> desc => <<"Client info as banned type">>, nullable => false, example => <<"Badass坏"/utf8>>})}, - {by, hoconsc:mk(binary(), #{ + {by, hoconsc:mk(emqx_schema:unicode_binary(), #{ desc => <<"Commander">>, nullable => true, example => <<"mgmt_api">>})}, @@ -113,15 +113,13 @@ fields(ban) -> desc => <<"Banned reason">>, nullable => true, example => <<"Too many requests">>})}, - {at, hoconsc:mk(binary(), #{ + {at, hoconsc:mk(emqx_schema:rfc3339_system_time(), #{ desc => <<"Create banned time, rfc3339, now if not specified">>, nullable => true, - validator => fun is_rfc3339/1, example => <<"2021-10-25T21:48:47+08:00">>})}, - {until, hoconsc:mk(binary(), #{ + {until, hoconsc:mk(emqx_schema:rfc3339_system_time(), #{ desc => <<"Cancel banned time, rfc3339, now + 5 minute if not specified">>, nullable => true, - validator => fun is_rfc3339/1, example => <<"2021-10-25T21:53:47+08:00">>}) } ]; @@ -130,22 +128,19 @@ fields(meta) -> emqx_dashboard_swagger:fields(limit) ++ [{count, hoconsc:mk(integer(), #{example => 1})}]. -is_rfc3339(Time) -> - try - emqx_banned:to_timestamp(Time), - ok - catch _:_ -> {error, Time} - end. - banned(get, #{query_string := Params}) -> Response = emqx_mgmt_api:paginate(?TAB, Params, ?FORMAT_FUN), {200, Response}; banned(post, #{body := Body}) -> - case emqx_banned:create(emqx_banned:parse(Body)) of - {ok, Banned} -> - {200, format(Banned)}; - {error, {already_exist, Old}} -> - {400, #{code => 'ALREADY_EXISTED', message => format(Old)}} + case emqx_banned:parse(Body) of + {error, Reason} -> + {400, #{code => 'PARAMS_ERROR', message => list_to_binary(Reason)}}; + Ban -> + case emqx_banned:create(Ban) of + {ok, Banned} -> {200, format(Banned)}; + {error, {already_exist, Old}} -> + {400, #{code => 'ALREADY_EXISTED', message => format(Old)}} + end end. delete_banned(delete, #{bindings := Params}) -> diff --git a/apps/emqx_management/test/emqx_mgmt_banned_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_banned_api_SUITE.erl new file mode 100644 index 000000000..d7cb87bb8 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_banned_api_SUITE.erl @@ -0,0 +1,144 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_banned_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_mgmt_api_test_util:init_suite(), + Config. + +end_per_suite(_) -> + emqx_mgmt_api_test_util:end_suite(). + +t_create(_Config) -> + Now = erlang:system_time(second), + At = emqx_banned:to_rfc3339(Now), + Until = emqx_banned:to_rfc3339(Now + 3), + ClientId = <<"TestClient测试"/utf8>>, + By = <<"banned suite测试组"/utf8>>, + Reason = <<"test测试"/utf8>>, + As = <<"clientid">>, + ClientIdBanned = #{ + as => As, + who => ClientId, + by => By, + reason => Reason, + at => At, + until => Until + }, + {ok, ClientIdBannedRes} = create_banned(ClientIdBanned), + ?assertEqual(#{<<"as">> => As, + <<"at">> => At, + <<"by">> => By, + <<"reason">> => Reason, + <<"until">> => Until, + <<"who">> => ClientId + }, ClientIdBannedRes), + PeerHost = <<"192.168.2.13">>, + PeerHostBanned = #{ + as => <<"peerhost">>, + who => PeerHost, + by => By, + reason => Reason, + at => At, + until => Until + }, + {ok, PeerHostBannedRes} = create_banned(PeerHostBanned), + ?assertEqual(#{<<"as">> => <<"peerhost">>, + <<"at">> => At, + <<"by">> => By, + <<"reason">> => Reason, + <<"until">> => Until, + <<"who">> => PeerHost + }, PeerHostBannedRes), + {ok, #{<<"data">> := List}} = list_banned(), + Bans = lists:sort(lists:map(fun(#{<<"who">> := W, <<"as">> := A}) -> {A, W} end, List)), + ?assertEqual([{<<"clientid">>, ClientId}, {<<"peerhost">>, PeerHost}], Bans), + ok. + +t_create_failed(_Config) -> + Now = erlang:system_time(second), + At = emqx_banned:to_rfc3339(Now), + Until = emqx_banned:to_rfc3339(Now + 10), + Who = <<"BadHost"/utf8>>, + By = <<"banned suite测试组"/utf8>>, + Reason = <<"test测试"/utf8>>, + As = <<"peerhost">>, + BadPeerHost = #{ + as => As, + who => Who, + by => By, + reason => Reason, + at => At, + until => Until + }, + BadRequest = {error, {"HTTP/1.1", 400, "Bad Request"}}, + ?assertEqual(BadRequest, create_banned(BadPeerHost)), + Expired = BadPeerHost#{until => emqx_banned:to_rfc3339(Now - 1), + who => <<"127.0.0.1">>}, + ?assertEqual(BadRequest, create_banned(Expired)), + ok. + +t_delete(_Config) -> + Now = erlang:system_time(second), + At = emqx_banned:to_rfc3339(Now), + Until = emqx_banned:to_rfc3339(Now + 3), + Who = <<"TestClient-"/utf8>>, + By = <<"banned suite 中"/utf8>>, + Reason = <<"test测试"/utf8>>, + As = <<"clientid">>, + Banned = #{ + as => clientid, + who => Who, + by => By, + reason => Reason, + at => At, + until => Until + }, + {ok, _} = create_banned(Banned), + ?assertMatch({ok, _}, delete_banned(binary_to_list(As), binary_to_list(Who))), + ?assertMatch({error,{"HTTP/1.1",404,"Not Found"}}, + delete_banned(binary_to_list(As), binary_to_list(Who))), + ok. + +list_banned() -> + Path = emqx_mgmt_api_test_util:api_path(["banned"]), + case emqx_mgmt_api_test_util:request_api(get, Path) of + {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])}; + Error -> Error + end. + +create_banned(Banned) -> + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Path = emqx_mgmt_api_test_util:api_path(["banned"]), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Banned) of + {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + Error -> Error + end. + +delete_banned(As, Who) -> + DeletePath = emqx_mgmt_api_test_util:api_path(["banned", As, Who]), + emqx_mgmt_api_test_util:request_api(delete, DeletePath). + +to_rfc3339(Sec) -> + list_to_binary(calendar:system_time_to_rfc3339(Sec)). From 9769ddeb14c6ce0594efe4f354d4410528e765f9 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 24 Dec 2021 13:40:13 +0800 Subject: [PATCH 06/99] fix(authn): attempt to convert certificate content only when TLS is enabled --- apps/emqx_authn/src/emqx_authn_api.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 595eed1c1..64ce3e6a4 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -726,7 +726,7 @@ with_chain(ListenerID, Fun) -> create_authenticator(ConfKeyPath, ChainName, Config) -> case update_config(ConfKeyPath, {create_authenticator, ChainName, Config}) of {ok, #{post_config_update := #{emqx_authentication := #{id := ID}}, - raw_config := AuthenticatorsConfig}} -> + raw_config := AuthenticatorsConfig}} -> {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), {200, maps:put(id, ID, convert_certs(fill_defaults(AuthenticatorConfig)))}; {error, {_PrePostConfigUpdate, emqx_authentication, Reason}} -> @@ -872,7 +872,7 @@ fill_defaults(Configs) when is_list(Configs) -> fill_defaults(Config) -> emqx_authn:check_config(Config, #{only_fill_defaults => true}). -convert_certs(#{ssl := SSLOpts} = Config) -> +convert_certs(#{ssl := #{enable := true} = SSLOpts} = Config) -> NSSLOpts = lists:foldl(fun(K, Acc) -> case maps:get(K, Acc, undefined) of undefined -> Acc; From 72a27e9d8fe40eb124ee492e8b25b0c7e738006f Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 24 Dec 2021 13:44:28 +0800 Subject: [PATCH 07/99] fix: fix undefined function call --- apps/emqx_authz/src/emqx_authz.erl | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 97e0fb595..d02da0958 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -31,7 +31,6 @@ , lookup/0 , lookup/1 , move/2 - , move/3 , update/2 , authorize/5 ]). @@ -109,15 +108,12 @@ lookup(Type) -> {Source, _Front, _Rear} = take(Type), Source. -move(Type, Cmd) -> - move(Type, Cmd, #{}). - -move(Type, #{<<"before">> := Before}, Opts) -> - emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}, Opts); -move(Type, #{<<"after">> := After}, Opts) -> - emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))}, Opts); -move(Type, Position, Opts) -> - emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), Position}, Opts). +move(Type, #{<<"before">> := Before}) -> + emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}); +move(Type, #{<<"after">> := After}) -> + emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))}); +move(Type, Position) -> + emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), Position}). update({?CMD_REPLACE, Type}, Sources) -> emqx_authz_utils:update_config(?CONF_KEY_PATH, {{?CMD_REPLACE, type(Type)}, Sources}); From b80a01554b14442d777a7972aeac519d0afeb90e Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 24 Dec 2021 18:03:55 +0800 Subject: [PATCH 08/99] fix(emqx_retainer): add support for RAP falg --- apps/emqx_retainer/src/emqx_retainer.erl | 48 +++++++++++++----------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 9be449b60..24dd05524 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -27,10 +27,10 @@ , on_message_publish/2 ]). --export([ dispatch/4 +-export([ dispatch/5 , delete_message/2 , store_retained/2 - , deliver/5]). + , deliver/6]). -export([ get_expiry_time/1 , update_config/1 @@ -78,7 +78,7 @@ on_session_subscribed(_, _, #{share := ShareName}, _) when ShareName =/= undefin on_session_subscribed(_, Topic, #{rh := Rh} = Opts, Context) -> IsNew = maps:get(is_new, Opts, true), case Rh =:= 0 orelse (Rh =:= 1 andalso IsNew) of - true -> dispatch(Context, Topic); + true -> dispatch(Context, Topic, Opts); _ -> ok end. @@ -111,26 +111,26 @@ on_message_publish(Msg, _) -> start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). --spec dispatch(context(), pid(), topic(), cursor()) -> ok. -dispatch(Context, Pid, Topic, Cursor) -> +-spec dispatch(context(), pid(), topic(), emqx_types:subopts(), cursor()) -> ok. +dispatch(Context, Pid, Topic, Opts, Cursor) -> Mod = get_backend_module(), case Cursor =/= undefined orelse emqx_topic:wildcard(Topic) of false -> {ok, Result} = Mod:read_message(Context, Topic), - deliver(Result, Context, Pid, Topic, undefined); + deliver(Result, Context, Pid, Topic, Opts, undefined); true -> {ok, Result, NewCursor} = Mod:match_messages(Context, Topic, Cursor), - deliver(Result, Context, Pid, Topic, NewCursor) + deliver(Result, Context, Pid, Topic, Opts, NewCursor) end. -deliver([], Context, Pid, Topic, Cursor) -> +deliver([], Context, Pid, Topic, Opts, Cursor) -> case Cursor of undefined -> ok; _ -> - dispatch(Context, Pid, Topic, Cursor) + dispatch(Context, Pid, Topic, Opts, Cursor) end; -deliver(Result, #{context_id := Id} = Context, Pid, Topic, Cursor) -> +deliver(Result, #{context_id := Id} = Context, Pid, Topic, Opts, Cursor) -> case erlang:is_process_alive(Pid) of false -> ok; @@ -138,12 +138,12 @@ deliver(Result, #{context_id := Id} = Context, Pid, Topic, Cursor) -> #{msg_deliver_quota := MaxDeliverNum} = emqx:get_config([?APP, flow_control]), case MaxDeliverNum of 0 -> - _ = [Pid ! {deliver, Topic, Msg} || Msg <- Result], + _ = [Pid ! {deliver, Topic, handle_retain_opts(Opts, Msg)} || Msg <- Result], ok; _ -> - case do_deliver(Result, Id, Pid, Topic) of + case do_deliver(Result, Id, Pid, Topic, Opts) of ok -> - deliver([], Context, Pid, Topic, Cursor); + deliver([], Context, Pid, Topic, Opts, Cursor); abort -> ok end @@ -280,9 +280,9 @@ is_too_big(Size) -> Limit > 0 andalso (Size > Limit). %% @private -dispatch(Context, Topic) -> - emqx_retainer_pool:async_submit(fun ?MODULE:dispatch/4, - [Context, self(), Topic, undefined]). +dispatch(Context, Topic, Opts) -> + emqx_retainer_pool:async_submit(fun ?MODULE:dispatch/5, + [Context, self(), Topic, Opts, undefined]). -spec delete_message(context(), topic()) -> ok. delete_message(Context, Topic) -> @@ -305,16 +305,16 @@ clean(Context) -> Mod = get_backend_module(), Mod:clean(Context). --spec do_deliver(list(term()), pos_integer(), pid(), topic()) -> ok | abort. -do_deliver([Msg | T], Id, Pid, Topic) -> +-spec do_deliver(list(term()), pos_integer(), pid(), topic(), emqx_types:subopts()) -> ok | abort. +do_deliver([Msg | T], Id, Pid, Topic, Opts) -> case require_semaphore(?DELIVER_SEMAPHORE, Id) of true -> - Pid ! {deliver, Topic, Msg}, - do_deliver(T, Id, Pid, Topic); + Pid ! {deliver, Topic, handle_retain_opts(Opts, Msg)}, + do_deliver(T, Id, Pid, Topic, Opts); _ -> abort end; -do_deliver([], _, _, _) -> +do_deliver([], _, _, _, _) -> ok. -spec require_semaphore(semaphore(), pos_integer()) -> boolean(). @@ -484,3 +484,9 @@ load(Context) -> unload() -> emqx:unhook('message.publish', {?MODULE, on_message_publish}), emqx:unhook('session.subscribed', {?MODULE, on_session_subscribed}). + +handle_retain_opts(#{rap := 0}, Message) -> + emqx_message:set_header(retain, false, Message); + +handle_retain_opts(_, Message) -> + Message. From 0b099432c5a8c6841f976a25bf39e36e13df3b08 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 24 Dec 2021 19:20:36 +0800 Subject: [PATCH 09/99] fix(conf): can't include local/overide-conf file --- apps/emqx/src/emqx_config.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index f3e6e1366..b2154b429 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -264,6 +264,7 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) -> {error, Reason} -> ?SLOG(error, #{msg => failed_to_load_hocon_conf, reason => Reason, + pwd => file:get_cwd(), include_dirs => IncDir }), error(failed_to_load_hocon_conf) @@ -280,7 +281,7 @@ init_load(SchemaMod, RawConf) when is_map(RawConf) -> maps:with(RootNames, RawConfWithEnvs)). include_dirs() -> - [filename:join(emqx:data_dir(), "configs")]. + [filename:join(emqx:data_dir(), "configs") ++ "/"]. merge_envs(SchemaMod, RawConf) -> Opts = #{logger => fun(_, _) -> ok end, %% everything should have been logged already when check_config From 668180388c08c6655358cb92a0f9a7216e6f2907 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 20 Dec 2021 09:23:00 +0800 Subject: [PATCH 10/99] feat(trace): replace logger_formatter by emqx_trace_formatter --- apps/emqx/include/logger.hrl | 2 + apps/emqx/src/emqx_broker.erl | 3 +- apps/emqx/src/emqx_channel.erl | 2 +- apps/emqx/src/emqx_connection.erl | 15 ++- apps/emqx/src/emqx_packet.erl | 3 +- apps/emqx/src/emqx_trace/emqx_trace.erl | 101 ++++++++++++------ .../src/emqx_trace/emqx_trace_formatter.erl | 60 +++++++++++ .../src/emqx_trace/emqx_trace_handler.erl | 65 +++++------ apps/emqx/src/emqx_ws_connection.erl | 11 +- apps/emqx/test/emqx_trace_handler_SUITE.erl | 18 ++-- .../src/emqx_connector_http.erl | 5 +- .../src/emqx_connector_ldap.erl | 5 +- .../src/emqx_connector_mongo.erl | 5 +- .../src/emqx_connector_mqtt.erl | 4 +- .../src/emqx_connector_mysql.erl | 4 +- .../src/emqx_connector_pgsql.erl | 4 +- .../src/emqx_connector_redis.erl | 4 +- .../src/emqx_mgmt_api_trace.erl | 4 +- apps/emqx_management/src/emqx_mgmt_cli.erl | 27 +++-- .../src/emqx_rule_outputs.erl | 4 +- .../src/emqx_rule_runtime.erl | 2 +- .../src/emqx_rule_sqltester.erl | 2 +- 22 files changed, 214 insertions(+), 136 deletions(-) create mode 100644 apps/emqx/src/emqx_trace/emqx_trace_formatter.erl diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index c2ee5ab95..ecedfafe7 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -69,6 +69,8 @@ ok end). +-define(TRACE(Action, Meta, Msg), emqx_trace:log(Action, Meta, Msg)). + %% print to 'user' group leader -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)). -define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)). diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index a82ab9b45..6dce69136 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -205,8 +205,7 @@ publish(Msg) when is_record(Msg, message) -> emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'), case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of #message{headers = #{allow_publish := false}} -> - ?SLOG(debug, #{msg => "message_not_published", - payload => emqx_message:to_log_map(Msg)}), + ?TRACE("NotAllow", #{payload => emqx_message:to_log_map(Msg)}, "message_not_published"), []; Msg1 = #message{topic = Topic} -> emqx_persistent_session:persist_message(Msg1), diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index eb71aca58..7fe366ad1 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -292,7 +292,7 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) -> fun check_banned/2 ], ConnPkt, Channel#channel{conn_state = connecting}) of {ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} -> - ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]), + ?TRACE("RECV", #{}, Packet), NChannel1 = NChannel#channel{ will_msg = emqx_packet:will_msg(NConnPkt), alias_maximum = init_alias_maximum(NConnPkt, ClientInfo) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 6919c6ff8..b8a6b4b7b 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -449,14 +449,12 @@ handle_msg({'$gen_cast', Req}, State) -> {ok, NewState}; handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl -> - ?SLOG(debug, #{msg => "RECV_data", data => Data, transport => Inet}), Oct = iolist_size(Data), inc_counter(incoming_bytes, Oct), ok = emqx_metrics:inc('bytes.received', Oct), when_bytes_in(Oct, Data, State); handle_msg({quic, Data, _Sock, _, _, _}, State) -> - ?SLOG(debug, #{msg => "RECV_data", data => Data, transport => quic}), Oct = iolist_size(Data), inc_counter(incoming_bytes, Oct), ok = emqx_metrics:inc('bytes.received', Oct), @@ -528,7 +526,7 @@ handle_msg({connack, ConnAck}, State) -> handle_outgoing(ConnAck, State); handle_msg({close, Reason}, State) -> - ?SLOG(debug, #{msg => "force_socket_close", reason => Reason}), + ?TRACE("CLOSE", #{reason => Reason}, "force_socket_close"), handle_info({sock_closed, Reason}, close_socket(State)); handle_msg({event, connected}, State = #state{channel = Channel}) -> @@ -566,7 +564,8 @@ terminate(Reason, State = #state{channel = Channel, transport = Transport, Channel1 = emqx_channel:set_conn_state(disconnected, Channel), emqx_congestion:cancel_alarms(Socket, Transport, Channel1), emqx_channel:terminate(Reason, Channel1), - close_socket_ok(State) + close_socket_ok(State), + ?TRACE("TERMINATE", #{reason => Reason}, "terminated") catch E : C : S -> ?tp(warning, unclean_terminate, #{exception => E, context => C, stacktrace => S}) @@ -716,7 +715,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) -> handle_incoming(Packet, State) when is_record(Packet, mqtt_packet) -> ok = inc_incoming_stats(Packet), - ?SLOG(debug, #{msg => "RECV_packet", packet => emqx_packet:format(Packet)}), + ?TRACE("RECV", #{}, Packet), with_channel(handle_in, [Packet], State); handle_incoming(FrameError, State) -> @@ -760,10 +759,8 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) -> ok = emqx_metrics:inc('delivery.dropped.too_large'), ok = emqx_metrics:inc('delivery.dropped'), <<>>; - Data -> ?SLOG(debug, #{ - msg => "SEND_packet", - packet => emqx_packet:format(Packet) - }), + Data -> + ?TRACE("SEND", #{}, Packet), ok = inc_outgoing_stats(Packet), Data catch diff --git a/apps/emqx/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl index 60835d4ab..02165a6b5 100644 --- a/apps/emqx/src/emqx_packet.erl +++ b/apps/emqx/src/emqx_packet.erl @@ -453,7 +453,7 @@ format_variable(undefined, _) -> format_variable(Variable, undefined) -> format_variable(Variable); format_variable(Variable, Payload) -> - io_lib:format("~ts, Payload=~0p", [format_variable(Variable), Payload]). + io_lib:format("~ts, Payload=~ts", [format_variable(Variable), Payload]). format_variable(#mqtt_packet_connect{ proto_ver = ProtoVer, @@ -520,4 +520,3 @@ format_password(_Password) -> '******'. i(true) -> 1; i(false) -> 0; i(I) when is_integer(I) -> I. - diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 42e4d0baf..f4679e073 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -26,6 +26,7 @@ -export([ publish/1 , subscribe/3 , unsubscribe/2 + , log/3 ]). -export([ start_link/0 @@ -36,6 +37,7 @@ , delete/1 , clear/0 , update/2 + , check/0 ]). -export([ format/1 @@ -50,6 +52,8 @@ -define(TRACE, ?MODULE). -define(MAX_SIZE, 30). +-define(TRACE_FILTER, emqx_trace_filter). +-define(OWN_KEYS,[level,filters,filter_default,handlers]). -ifdef(TEST). -export([ log_file/2 @@ -80,27 +84,53 @@ mnesia(boot) -> publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore; publish(#message{from = From, topic = Topic, payload = Payload}) when is_binary(From); is_atom(From) -> - emqx_logger:info( - #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, - "PUBLISH to ~s: ~0p", - [Topic, Payload] - ). + ?TRACE("PUBLISH", #{topic => Topic}, {publish, Payload}). subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore; subscribe(Topic, SubId, SubOpts) -> - emqx_logger:info( - #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, - "~ts SUBSCRIBE ~ts: Options: ~0p", - [SubId, Topic, SubOpts] - ). + ?TRACE("SUBSCRIBE", #{topic => Topic}, {subscribe, SubId, SubOpts}). unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore; unsubscribe(Topic, SubOpts) -> - emqx_logger:info( - #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, - "~ts UNSUBSCRIBE ~ts: Options: ~0p", - [maps:get(subid, SubOpts, ""), Topic, SubOpts] - ). + ?TRACE("UNSUBSCRIBE", #{topic => Topic}, {unsubscribe, SubOpts}). + +log(Action, Meta0, Msg) -> + case persistent_term:get(?TRACE_FILTER, undefined) of + undefined -> ok; + List -> + Meta = maps:merge(logger:get_process_metadata(), Meta0), + Log = #{level => trace, action => Action, meta => Meta, msg => Msg}, + log_filter(List, Log) + end. + +log_filter([], _Log) -> ok; +log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) -> + case FilterFun(Log0, {Filter, Name}) of + stop -> stop; + ignore -> ignore; + Log -> + case logger_config:get(ets:whereis(logger), Id) of + {ok, #{module := Module} = HandlerConfig0} -> + HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0), + try Module:log(Log, HandlerConfig) + catch C:R:S -> + case logger:remove_handler(Id) of + ok -> + logger:internal_log(error, {removed_failing_handler, Id}); + {error,{not_found,_}} -> + %% Probably already removed by other client + %% Don't report again + ok; + {error,Reason} -> + logger:internal_log(error, + {removed_handler_failed, Id, Reason, C, R, S}) + end + end; + {error, {not_found, Id}} -> ok; + {error, Reason} -> logger:internal_log(error, {find_handle_id_failed, Id, Reason}) + end + end, + log_filter(Rest, Log0). -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> @@ -161,6 +191,9 @@ update(Name, Enable) -> end, transaction(Tran). +check() -> + erlang:send(?MODULE, {mnesia_table_event, check}). + -spec get_trace_filename(Name :: binary()) -> {ok, FileName :: string()} | {error, not_found}. get_trace_filename(Name) -> @@ -196,14 +229,13 @@ format(Traces) -> init([]) -> ok = mria:wait_for_tables([?TRACE]), erlang:process_flag(trap_exit, true), - OriginLogLevel = emqx_logger:get_primary_log_level(), ok = filelib:ensure_dir(trace_dir()), ok = filelib:ensure_dir(zip_dir()), {ok, _} = mnesia:subscribe({table, ?TRACE, simple}), Traces = get_enable_trace(), - ok = update_log_primary_level(Traces, OriginLogLevel), TRef = update_trace(Traces), - {ok, #{timer => TRef, monitors => #{}, primary_log_level => OriginLogLevel}}. + update_trace_handler(), + {ok, #{timer => TRef, monitors => #{}}}. handle_call(Req, _From, State) -> ?SLOG(error, #{unexpected_call => Req}), @@ -224,10 +256,10 @@ handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitor {noreply, State#{monitors => NewMonitors}} end; handle_info({timeout, TRef, update_trace}, - #{timer := TRef, primary_log_level := OriginLogLevel} = State) -> + #{timer := TRef} = State) -> Traces = get_enable_trace(), - ok = update_log_primary_level(Traces, OriginLogLevel), NextTRef = update_trace(Traces), + update_trace_handler(), {noreply, State#{timer => NextTRef}}; handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) -> @@ -238,11 +270,11 @@ handle_info(Info, State) -> ?SLOG(error, #{unexpected_info => Info}), {noreply, State}. -terminate(_Reason, #{timer := TRef, primary_log_level := OriginLogLevel}) -> - ok = set_log_primary_level(OriginLogLevel), +terminate(_Reason, #{timer := TRef}) -> _ = mnesia:unsubscribe({table, ?TRACE, simple}), emqx_misc:cancel_timer(TRef), stop_all_trace_handler(), + update_trace_handler(), _ = file:del_dir_r(zip_dir()), ok. @@ -270,7 +302,7 @@ update_trace(Traces) -> disable_finished(Finished), Started = emqx_trace_handler:running(), {NeedRunning, AllStarted} = start_trace(Running, Started), - NeedStop = AllStarted -- NeedRunning, + NeedStop = filter_cli_handler(AllStarted) -- NeedRunning, ok = stop_trace(NeedStop, Started), clean_stale_trace_files(), NextTime = find_closest_time(Traces, Now), @@ -481,11 +513,20 @@ transaction(Tran) -> {aborted, Reason} -> {error, Reason} end. -update_log_primary_level([], OriginLevel) -> set_log_primary_level(OriginLevel); -update_log_primary_level(_, _) -> set_log_primary_level(debug). - -set_log_primary_level(NewLevel) -> - case NewLevel =/= emqx_logger:get_primary_log_level() of - true -> emqx_logger:set_primary_log_level(NewLevel); - false -> ok +update_trace_handler() -> + case emqx_trace_handler:running() of + [] -> persistent_term:erase(?TRACE_FILTER); + Running -> + List = lists:map(fun(#{id := Id, filter_fun := FilterFun, + filter := Filter, name := Name}) -> + {Id, FilterFun, Filter, Name} end, Running), + case List =/= persistent_term:get(?TRACE_FILTER, undefined) of + true -> persistent_term:put(?TRACE_FILTER, List); + false -> ok + end end. + +filter_cli_handler(Names) -> + lists:filter(fun(Name) -> + notmatch =:= re:run(Name, "^CLI-+.", []) + end, Names). diff --git a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl new file mode 100644 index 000000000..1b37b7130 --- /dev/null +++ b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl @@ -0,0 +1,60 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_trace_formatter). + +-export([format/2]). + +%%%----------------------------------------------------------------- +%%% API +-spec format(LogEvent, Config) -> unicode:chardata() when + LogEvent :: logger:log_event(), + Config :: logger:config(). +format(#{level := trace, msg := Msg, meta := Meta, action := Action}, _Config) -> + Time = calendar:system_time_to_rfc3339(erlang:system_time(second)), + ClientId = maps:get(clientid, Meta, ""), + Peername = maps:get(peername, Meta, ""), + MsgBin = format_msg(Msg), + MetaBin = format_map(maps:without([clientid, peername], Meta)), + [Time, " [", Action, "] ", ClientId, "@", Peername, " ", MsgBin, " ( ", + MetaBin, ")\n"]; + +format(Event, Config) -> + emqx_logger_textfmt:format(Event, Config). + +format_msg(Bin)when is_binary(Bin) -> Bin; +format_msg(List) when is_list(List) -> List; +format_msg({publish, Payload}) -> + io_lib:format("Publish Payload:(~ts) TO ", [Payload]); +format_msg({subscribe, SubId, SubOpts}) -> + [io_lib:format("SUBSCRIBE ~ts, Opts( ", [SubId]), + format_map(SubOpts), ")"]; +format_msg({unsubscribe, SubOpts}) -> + [io_lib:format("UNSUBSCRIBE ~ts, Opts( ", [maps:get(subid, SubOpts, "undefined")]), + format_map(maps:without([subid], SubOpts)), ")"]; +format_msg(Packet) -> + emqx_packet:format(Packet). + +format_map(Map) -> + maps:fold(fun(K, V, Acc) -> + [to_iolist(K), ":", to_iolist(V), " "|Acc] + end, [], Map). + +to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom); +to_iolist(Int) when is_integer(Int) -> integer_to_list(Int); +to_iolist(Float) when is_float(Float) -> float_to_list(Float, [{decimals, 2}]); +to_iolist(Bin)when is_binary(Bin) -> unicode:characters_to_binary(Bin); +to_iolist(List) when is_list(List) -> unicode:characters_to_list(List); +to_iolist(Term) -> io_lib:format("~0p", [Term]). diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl index c76bf1aa9..9c991301c 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl @@ -25,6 +25,7 @@ -export([ running/0 , install/3 , install/4 + , install/5 , uninstall/1 , uninstall/2 ]). @@ -77,22 +78,18 @@ install(Type, Filter, Level, LogFile) -> -spec install(tracer(), logger:level() | all, string()) -> ok | {error, term()}. install(Who, all, LogFile) -> install(Who, debug, LogFile); -install(Who, Level, LogFile) -> - PrimaryLevel = emqx_logger:get_primary_log_level(), - try logger:compare_levels(Level, PrimaryLevel) of - lt -> - {error, - io_lib:format( - "Cannot trace at a log level (~s) " - "lower than the primary log level (~s)", - [Level, PrimaryLevel] - )}; - _GtOrEq -> - install_handler(Who, Level, LogFile) - catch - error:badarg -> - {error, {invalid_log_level, Level}} - end. +install(Who = #{name := Name, type := Type}, Level, LogFile) -> + HandlerId = handler_id(Name, Type), + Config = #{ + level => Level, + formatter => formatter(Who), + filter_default => stop, + filters => filters(Who), + config => ?CONFIG(LogFile) + }, + Res = logger:add_handler(HandlerId, logger_disk_log_h, Config), + show_prompts(Res, Who, "start_trace"), + Res. -spec uninstall(Type :: clientid | topic | ip_address, Name :: binary() | list()) -> ok | {error, term()}. @@ -121,38 +118,25 @@ uninstall(HandlerId) -> running() -> lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)). --spec filter_clientid(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. +-spec filter_clientid(logger:log_event(), {string(), atom()}) -> logger:log_event() | stop. filter_clientid(#{meta := #{clientid := ClientId}} = Log, {ClientId, _Name}) -> Log; -filter_clientid(_Log, _ExpectId) -> ignore. +filter_clientid(_Log, _ExpectId) -> stop. --spec filter_topic(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. +-spec filter_topic(logger:log_event(), {string(), atom()}) -> logger:log_event() | stop. filter_topic(#{meta := #{topic := Topic}} = Log, {TopicFilter, _Name}) -> case emqx_topic:match(Topic, TopicFilter) of true -> Log; - false -> ignore + false -> stop end; -filter_topic(_Log, _ExpectId) -> ignore. +filter_topic(_Log, _ExpectId) -> stop. --spec filter_ip_address(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. +-spec filter_ip_address(logger:log_event(), {string(), atom()}) -> logger:log_event() | stop. filter_ip_address(#{meta := #{peername := Peername}} = Log, {IP, _Name}) -> case lists:prefix(IP, Peername) of true -> Log; - false -> ignore + false -> stop end; -filter_ip_address(_Log, _ExpectId) -> ignore. - -install_handler(Who = #{name := Name, type := Type}, Level, LogFile) -> - HandlerId = handler_id(Name, Type), - Config = #{ - level => Level, - formatter => formatter(Who), - filter_default => stop, - filters => filters(Who), - config => ?CONFIG(LogFile) - }, - Res = logger:add_handler(HandlerId, logger_disk_log_h, Config), - show_prompts(Res, Who, "Start trace"), - Res. +filter_ip_address(_Log, _ExpectId) -> stop. filters(#{type := clientid, filter := Filter, name := Name}) -> [{clientid, {fun ?MODULE:filter_clientid/2, {ensure_list(Filter), Name}}}]; @@ -162,7 +146,7 @@ filters(#{type := ip_address, filter := Filter, name := Name}) -> [{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}]. formatter(#{type := Type}) -> - {logger_formatter, + {emqx_trace_formatter, #{ template => template(Type), single_line => false, @@ -176,7 +160,6 @@ formatter(#{type := Type}) -> %% (actually should use `~ts`), the utf8 characters clientid will become very difficult to read. template(clientid) -> [time, " [", level, "] ", {peername, [peername, " "], []}, msg, "\n"]; -%% TODO better format when clientid is utf8. template(_) -> [time, " [", level, "] ", {clientid, @@ -189,11 +172,11 @@ template(_) -> filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) -> Init = #{id => Id, level => Level, dst => Dst}, case Filters of - [{Type, {_FilterFun, {Filter, Name}}}] when + [{Type, {FilterFun, {Filter, Name}}}] when Type =:= topic orelse Type =:= clientid orelse Type =:= ip_address -> - [Init#{type => Type, filter => Filter, name => Name} | Acc]; + [Init#{type => Type, filter => Filter, name => Name, filter_fun => FilterFun} | Acc]; _ -> Acc end. diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index 375b1ae2f..6e6bc1c90 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -347,7 +347,6 @@ websocket_handle({binary, Data}, State) when is_list(Data) -> websocket_handle({binary, iolist_to_binary(Data)}, State); websocket_handle({binary, Data}, State) -> - ?SLOG(debug, #{msg => "RECV_data", data => Data, transport => websocket}), State2 = ensure_stats_timer(State), {Packets, State3} = parse_incoming(Data, [], State2), LenMsg = erlang:length(Packets), @@ -432,11 +431,11 @@ websocket_info(Info, State) -> websocket_close({_, ReasonCode, _Payload}, State) when is_integer(ReasonCode) -> websocket_close(ReasonCode, State); websocket_close(Reason, State) -> - ?SLOG(debug, #{msg => "websocket_closed", reason => Reason}), + ?TRACE("CLOSED", #{transport => websocket, reason => Reason}, "websocket_closed"), handle_info({sock_closed, Reason}, State). terminate(Reason, _Req, #state{channel = Channel}) -> - ?SLOG(debug, #{msg => "terminated", reason => Reason}), + ?TRACE("TERMINATE", #{transport => websocket, reason => Reason}, "webscoket_terminated"), emqx_channel:terminate(Reason, Channel); terminate(_Reason, _Req, _UnExpectedState) -> @@ -480,7 +479,7 @@ handle_info({connack, ConnAck}, State) -> return(enqueue(ConnAck, State)); handle_info({close, Reason}, State) -> - ?SLOG(debug, #{msg => "force_socket_close", reason => Reason}), + ?TRACE("CLOSE", #{reason => Reason}, "force_socket_close"), return(enqueue({close, Reason}, State)); handle_info({event, connected}, State = #state{channel = Channel}) -> @@ -663,7 +662,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) -> handle_incoming(Packet, State = #state{listener = {Type, Listener}}) when is_record(Packet, mqtt_packet) -> - ?SLOG(debug, #{msg => "RECV", packet => emqx_packet:format(Packet)}), + ?TRACE("RECV", #{transport => websocket}, Packet), ok = inc_incoming_stats(Packet), NState = case emqx_pd:get_counter(incoming_pubs) > get_active_n(Type, Listener) of @@ -727,7 +726,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) -> ok = emqx_metrics:inc('delivery.dropped.too_large'), ok = emqx_metrics:inc('delivery.dropped'), <<>>; - Data -> ?SLOG(debug, #{msg => "SEND", packet => Packet}), + Data -> ?TRACE("SEND", #{transport => websocket}, Packet), ok = inc_outgoing_stats(Packet), Data catch diff --git a/apps/emqx/test/emqx_trace_handler_SUITE.erl b/apps/emqx/test/emqx_trace_handler_SUITE.erl index abe233b58..01d587028 100644 --- a/apps/emqx/test/emqx_trace_handler_SUITE.erl +++ b/apps/emqx/test/emqx_trace_handler_SUITE.erl @@ -32,36 +32,30 @@ all() -> [t_trace_clientid, t_trace_topic, t_trace_ip_address, t_trace_clientid_ init_per_suite(Config) -> emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), + emqx_common_test_helpers:start_apps([emqx_modules]), Config. end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([emqx_modules]). init_per_testcase(t_trace_clientid, Config) -> Config; init_per_testcase(_Case, Config) -> - ok = emqx_logger:set_log_level(debug), _ = [logger:remove_handler(Id) ||#{id := Id} <- emqx_trace_handler:running()], Config. end_per_testcase(_Case, _Config) -> - ok = emqx_logger:set_log_level(warning), ok. t_trace_clientid(_Config) -> %% Start tracing - emqx_logger:set_log_level(error), - {error, _} = emqx_trace_handler:install(clientid, <<"client">>, debug, "tmp/client.log"), - emqx_logger:set_log_level(debug), %% add list clientid ok = emqx_trace_handler:install(clientid, "client", debug, "tmp/client.log"), ok = emqx_trace_handler:install(clientid, <<"client2">>, all, "tmp/client2.log"), ok = emqx_trace_handler:install(clientid, <<"client3">>, all, "tmp/client3.log"), - {error, {invalid_log_level, bad_level}} = - emqx_trace_handler:install(clientid, <<"client4">>, bad_level, "tmp/client4.log"), {error, {handler_not_added, {file_error, ".", eisdir}}} = emqx_trace_handler:install(clientid, <<"client5">>, debug, "."), + emqx_trace:check(), ok = filesync(<<"client">>, clientid), ok = filesync(<<"client2">>, clientid), ok = filesync(<<"client3">>, clientid), @@ -106,10 +100,9 @@ t_trace_clientid(_Config) -> ?assertEqual([], emqx_trace_handler:running()). t_trace_clientid_utf8(_) -> - emqx_logger:set_log_level(debug), - Utf8Id = <<"client 漢字編碼"/utf8>>, ok = emqx_trace_handler:install(clientid, Utf8Id, debug, "tmp/client-utf8.log"), + emqx_trace:check(), {ok, T} = emqtt:start_link([{clientid, Utf8Id}]), emqtt:connect(T), [begin emqtt:publish(T, <<"a/b/c">>, <<"hi">>) end|| _ <- lists:seq(1, 10)], @@ -126,9 +119,9 @@ t_trace_topic(_Config) -> emqtt:connect(T), %% Start tracing - emqx_logger:set_log_level(debug), ok = emqx_trace_handler:install(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"), ok = emqx_trace_handler:install(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"), + emqx_trace:check(), ok = filesync(<<"x/#">>, topic), ok = filesync(<<"y/#">>, topic), @@ -174,6 +167,7 @@ t_trace_ip_address(_Config) -> %% Start tracing ok = emqx_trace_handler:install(ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"), ok = emqx_trace_handler:install(ip_address, "192.168.1.1", all, "tmp/ip_trace_y.log"), + emqx_trace:check(), ok = filesync(<<"127.0.0.1">>, ip_address), ok = filesync(<<"192.168.1.1">>, ip_address), diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 2b9bd48aa..b54d87f12 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -199,9 +199,8 @@ on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) -> on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State); on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery, #{pool_name := PoolName, base_path := BasePath} = State) -> - ?SLOG(debug, #{msg => "http connector received request", - request => Request, connector => InstId, - state => State}), + ?TRACE("QUERY", #{request => Request, connector => InstId, state => State}, + "http connector received request"), NRequest = update_path(BasePath, Request), case Result = ehttpc:request(case KeyOrNum of undefined -> PoolName; diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index 8af516b82..8aa1f9319 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -87,9 +87,8 @@ on_stop(InstId, #{poolname := PoolName}) -> on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) -> Request = {Base, Filter, Attributes}, - ?SLOG(debug, #{msg => "ldap connector received request", - request => Request, connector => InstId, - state => State}), + ?TRACE("QUERY", #{request => Request, connector => InstId, state => State}, + "ldap connector received request"), case Result = ecpool:pick_and_do( PoolName, {?MODULE, search, [Base, Filter, Attributes]}, diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 6a1b15e57..d2594ab93 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -137,9 +137,8 @@ on_query(InstId, AfterQuery, #{poolname := PoolName} = State) -> Request = {Action, Collection, Selector, Docs}, - ?SLOG(debug, #{msg => "mongodb connector received request", - request => Request, connector => InstId, - state => State}), + ?TRACE("QUERY", #{request => Request, connector => InstId, state => State}, + "mongodb connector received request"), case ecpool:pick_and_do(PoolName, {?MODULE, mongo_query, [Action, Collection, Selector, Docs]}, no_handover) of diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index f8d17ce32..6d620cc14 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -150,8 +150,8 @@ on_stop(_InstId, #{name := InstanceId}) -> end. on_query(_InstId, {send_message, Msg}, AfterQuery, #{name := InstanceId}) -> - ?SLOG(debug, #{msg => "send msg to remote node", message => Msg, - connector => InstanceId}), + ?TRACE("QUERY", #{message => Msg, connector => InstanceId}, + "send msg to remote node"), emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg), emqx_resource:query_success(AfterQuery). diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index c93a1e350..def3904b4 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -85,8 +85,8 @@ on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) -> on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := _PoolName} = State) -> on_query(InstId, {sql, SQL, Params, default_timeout}, AfterQuery, State); on_query(InstId, {sql, SQL, Params, Timeout}, AfterQuery, #{poolname := PoolName} = State) -> - ?SLOG(debug, #{msg => "mysql connector received sql query", - connector => InstId, sql => SQL, state => State}), + ?TRACE("QUERY", #{connector => InstId, sql => SQL, state => State}, + "mysql connector received sql query"), case Result = ecpool:pick_and_do( PoolName, {mysql, query, [SQL, Params, Timeout]}, diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index f42bed666..ac864a45d 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -83,8 +83,8 @@ on_stop(InstId, #{poolname := PoolName}) -> on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) -> on_query(InstId, {sql, SQL, []}, AfterQuery, State); on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) -> - ?SLOG(debug, #{msg => "postgresql connector received sql query", - connector => InstId, sql => SQL, state => State}), + ?TRACE("QUERY", #{connector => InstId, sql => SQL, state => State}, + "postgresql connector received sql query"), case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [SQL, Params]}, no_handover) of {error, Reason} -> ?SLOG(error, #{ diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 075ede0bc..61b716b8b 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -125,8 +125,8 @@ on_stop(InstId, #{poolname := PoolName}) -> emqx_plugin_libs_pool:stop_pool(PoolName). on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := Type} = State) -> - ?SLOG(debug, #{msg => "redis connector received cmd query", - connector => InstId, sql => Command, state => State}), + ?TRACE("QUERY", #{connector => InstId, sql => Command, state => State}, + "redis connector received cmd query"), Result = case Type of cluster -> eredis_cluster:q(PoolName, Command); _ -> ecpool:pick_and_do(PoolName, {?MODULE, cmd, [Type, Command]}, no_handover) diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index d6902d123..5669e5653 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -142,11 +142,11 @@ fields(trace) -> #{desc => """Filter type""", nullable => false, example => <<"clientid">>})}, - {topic, hoconsc:mk(binary(), + {topic, hoconsc:mk(emqx_schema:unicode_binary(), #{desc => """support mqtt wildcard topic.""", nullable => true, example => <<"/dev/#">>})}, - {clientid, hoconsc:mk(binary(), + {clientid, hoconsc:mk(emqx_schema:unicode_binary(), #{desc => """mqtt clientid.""", nullable => true, example => <<"dev-001">>})}, diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 5885a2b17..0084855ab 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -395,9 +395,11 @@ trace(["stop", Operation, ClientId]) -> trace(["start", Operation, ClientId, LogFile]) -> trace(["start", Operation, ClientId, LogFile, "all"]); -trace(["start", Operation, ClientId, LogFile, Level]) -> +trace(["start", Operation, Filter, LogFile, Level]) -> case trace_type(Operation) of - {ok, Type} -> trace_on(Type, ClientId, list_to_existing_atom(Level), LogFile); + {ok, Type} -> + trace_on(name(Filter), Type, Filter, + list_to_existing_atom(Level), LogFile); error -> trace([]) end; @@ -417,20 +419,22 @@ trace(_) -> "Stop tracing for a client ip on local node"} ]). -trace_on(Who, Name, Level, LogFile) -> - case emqx_trace_handler:install(Who, Name, Level, LogFile) of +trace_on(Name, Type, Filter, Level, LogFile) -> + case emqx_trace_handler:install(Name, Type, Filter, Level, LogFile) of ok -> - emqx_ctl:print("trace ~s ~s successfully~n", [Who, Name]); + emqx_trace:check(), + emqx_ctl:print("trace ~s ~s successfully~n", [Filter, Name]); {error, Error} -> - emqx_ctl:print("[error] trace ~s ~s: ~p~n", [Who, Name, Error]) + emqx_ctl:print("[error] trace ~s ~s: ~p~n", [Filter, Name, Error]) end. -trace_off(Who, Name) -> - case emqx_trace_handler:uninstall(Who, Name) of +trace_off(Who, Filter) -> + case emqx_trace_handler:uninstall(Who, name(Filter)) of ok -> - emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Name]); + emqx_trace:check(), + emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Filter]); {error, Error} -> - emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Who, Name, Error]) + emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Who, Filter, Error]) end. %%-------------------------------------------------------------------- @@ -716,3 +720,6 @@ format_listen_on({Addr, Port}) when is_list(Addr) -> io_lib:format("~ts:~w", [Addr, Port]); format_listen_on({Addr, Port}) when is_tuple(Addr) -> io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]). + +name(Filter) -> + iolist_to_binary(["CLI-", Filter]). diff --git a/apps/emqx_rule_engine/src/emqx_rule_outputs.erl b/apps/emqx_rule_engine/src/emqx_rule_outputs.erl index 61a520e81..70aa68cf5 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_outputs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_outputs.erl @@ -85,7 +85,7 @@ republish(Selected, #{flags := Flags, metadata := #{rule_id := RuleId}}, Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected), QoS = replace_simple_var(QoSTks, Selected, 0), Retain = replace_simple_var(RetainTks, Selected, false), - ?SLOG(debug, #{msg => "republish", topic => Topic, payload => Payload}), + ?TRACE("REPUBLISH", #{topic => Topic, payload => Payload}, "republish message"), safe_publish(RuleId, Topic, QoS, Flags#{retain => Retain}, Payload); %% in case this is a "$events/" event @@ -99,7 +99,7 @@ republish(Selected, #{metadata := #{rule_id := RuleId}}, Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected), QoS = replace_simple_var(QoSTks, Selected, 0), Retain = replace_simple_var(RetainTks, Selected, false), - ?SLOG(debug, #{msg => "republish", topic => Topic, payload => Payload}), + ?TRACE("REPUBLISH", #{topic => Topic, payload => Payload}, "republish"), safe_publish(RuleId, Topic, QoS, #{retain => Retain}, Payload). %%-------------------------------------------------------------------- diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 4225c6f72..c61296d87 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -248,7 +248,7 @@ handle_output(OutId, Selected, Envs) -> end. do_handle_output(BridgeId, Selected, _Envs) when is_binary(BridgeId) -> - ?SLOG(debug, #{msg => "output to bridge", bridge_id => BridgeId}), + ?TRACE("SEND", #{bridge_id => BridgeId}, "output to bridge"), emqx_bridge:send_message(BridgeId, Selected); do_handle_output(#{mod := Mod, func := Func, args := Args}, Selected, Envs) -> Mod:Func(Selected, Envs, Args). diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index 74ec1bb1c..7a9da25a2 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -77,7 +77,7 @@ flatten([D1 | L]) when is_list(D1) -> D1 ++ flatten(L). echo_action(Data, Envs) -> - ?SLOG(debug, #{msg => "testing_rule_sql_ok", data => Data, envs => Envs}), + ?TRACE("TEST", #{data => Data, envs => Envs}, "testing_rule_sql_ok"), Data. fill_default_values(Event, Context) -> From fca5a3bc21eafe85cfbab57e290f202aa9f9efa9 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 24 Dec 2021 11:34:01 +0800 Subject: [PATCH 11/99] chore(gw): add cli testcases --- .../src/emqx_gateway_api_clients.erl | 16 +- apps/emqx_gateway/src/emqx_gateway_cli.erl | 54 ++++--- .../test/emqx_gateway_cli_SUITE.erl | 150 ++++++++++++++++++ 3 files changed, 195 insertions(+), 25 deletions(-) create mode 100644 apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index 697bccc1d..3c902ac8d 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -532,7 +532,21 @@ params_client_searching_in_qs() -> , {lte_connected_at, mk(binary(), M#{desc => <<"Match the client socket connected datatime less than " - " a certain value">>})} + "a certain value">>})} + , {endpoint_name, + mk(binary(), + M#{desc => <<"Match the lwm2m client's endpoint name">>})} + , {like_endpoint_name, + mk(binary(), + M#{desc => <<"Use sub-string to match lwm2m client's endpoint name">>})} + , {gte_lifetime, + mk(binary(), + M#{desc => <<"Match the lwm2m client registered lifetime greater " + "than a certain value">>})} + , {lte_lifetime, + mk(binary(), + M#{desc => <<"Match the lwm2m client registered lifetime less than " + "a certain value">>})} ]. params_paging() -> diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl index 03d55e27e..030c0057b 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cli.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -50,18 +50,24 @@ is_cmd(Fun) -> %% Cmds gateway(["list"]) -> - lists:foreach(fun(#{name := Name} = Gateway) -> - %% TODO: More infos: listeners?, connected? - Status = maps:get(status, Gateway, stopped), - print("Gateway(name=~ts, status=~ts)~n", [Name, Status]) - end, emqx_gateway:list()); + lists:foreach( + fun (#{name := Name, status := unloaded}) -> + print("Gateway(name=~ts, status=unloaded)\n", [Name]); + (#{name := Name, status := stopped, stopped_at := StoppedAt}) -> + print("Gateway(name=~ts, status=stopped, stopped_at=~ts)\n", + [Name, StoppedAt]); + (#{name := Name, status := running, current_connections := ConnCnt, + started_at := StartedAt}) -> + print("Gateway(name=~ts, status=running, clients=~w, started_at=~ts)\n", + [Name, ConnCnt, StartedAt]) + end, emqx_gateway_http:gateways(all)); gateway(["lookup", Name]) -> case emqx_gateway:lookup(atom(Name)) of undefined -> - print("undefined~n"); + print("undefined\n"); Info -> - print("~p~n", [Info]) + print("~p\n", [Info]) end; gateway(["load", Name, Conf]) -> @@ -70,17 +76,17 @@ gateway(["load", Name, Conf]) -> emqx_json:decode(Conf, [return_maps]) ) of {ok, _} -> - print("ok~n"); + print("ok\n"); {error, Reason} -> - print("Error: ~p~n", [Reason]) + print("Error: ~p\n", [Reason]) end; gateway(["unload", Name]) -> case emqx_gateway_conf:unload_gateway(bin(Name)) of ok -> - print("ok~n"); + print("ok\n"); {error, Reason} -> - print("Error: ~p~n", [Reason]) + print("Error: ~p\n", [Reason]) end; gateway(["stop", Name]) -> @@ -89,9 +95,9 @@ gateway(["stop", Name]) -> #{<<"enable">> => <<"false">>} ) of {ok, _} -> - print("ok~n"); + print("ok\n"); {error, Reason} -> - print("Error: ~p~n", [Reason]) + print("Error: ~p\n", [Reason]) end; gateway(["start", Name]) -> @@ -100,9 +106,9 @@ gateway(["start", Name]) -> #{<<"enable">> => <<"true">>} ) of {ok, _} -> - print("ok~n"); + print("ok\n"); {error, Reason} -> - print("Error: ~p~n", [Reason]) + print("Error: ~p\n", [Reason]) end; gateway(_) -> @@ -123,7 +129,7 @@ gateway(_) -> 'gateway-registry'(["list"]) -> lists:foreach( fun({Name, #{cbkmod := CbMod}}) -> - print("Registered Name: ~ts, Callback Module: ~ts~n", [Name, CbMod]) + print("Registered Name: ~ts, Callback Module: ~ts\n", [Name, CbMod]) end, emqx_gateway_registry:list()); @@ -137,15 +143,15 @@ gateway(_) -> InfoTab = emqx_gateway_cm:tabname(info, Name), case ets:info(InfoTab) of undefined -> - print("Bad Gateway Name.~n"); + print("Bad Gateway Name.\n"); _ -> - dump(InfoTab, client) + dump(InfoTab, client) end; 'gateway-clients'(["lookup", Name, ClientId]) -> ChanTab = emqx_gateway_cm:tabname(chan, Name), case ets:lookup(ChanTab, bin(ClientId)) of - [] -> print("Not Found.~n"); + [] -> print("Not Found.\n"); [Chann] -> InfoTab = emqx_gateway_cm:tabname(info, Name), [ChannInfo] = ets:lookup(InfoTab, Chann), @@ -154,8 +160,8 @@ gateway(_) -> 'gateway-clients'(["kick", Name, ClientId]) -> case emqx_gateway_cm:kick_session(Name, bin(ClientId)) of - ok -> print("ok~n"); - _ -> print("Not Found.~n") + ok -> print("ok\n"); + _ -> print("Not Found.\n") end; 'gateway-clients'(_) -> @@ -171,11 +177,11 @@ gateway(_) -> Tab = emqx_gateway_metrics:tabname(Name), case ets:info(Tab) of undefined -> - print("Bad Gateway Name.~n"); + print("Bad Gateway Name.\n"); _ -> lists:foreach( fun({K, V}) -> - print("~-30s: ~w~n", [K, V]) + print("~-30s: ~w\n", [K, V]) end, lists:sort(ets:tab2list(Tab))) end; @@ -232,7 +238,7 @@ print_record({client, {_, Infos, Stats}}) -> print("Client(~ts, username=~ts, peername=~ts, " "clean_start=~ts, keepalive=~w, " "subscriptions=~w, delivered_msgs=~w, " - "connected=~ts, created_at=~w, connected_at=~w)~n", + "connected=~ts, created_at=~w, connected_at=~w)\n", [format(K, maps:get(K, Info)) || K <- InfoKeys]). print(S) -> emqx_ctl:print(S). diff --git a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl new file mode 100644 index 000000000..a2338ea26 --- /dev/null +++ b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl @@ -0,0 +1,150 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_cli_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-define(GP(S), begin S, receive {fmt, P} -> P; O -> O end end). + +%% this parses to #{}, will not cause config cleanup +%% so we will need call emqx_config:erase +-define(CONF_DEFAULT, <<" +gateway {} +">>). + +%%-------------------------------------------------------------------- +%% Setup +%%-------------------------------------------------------------------- + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Conf) -> + emqx_config:erase(gateway), + emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), + emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]), + Conf. + +end_per_suite(Conf) -> + emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_authn, emqx_conf]), + Conf. + +init_per_testcase(_, Conf) -> + Self = self(), + ok = meck:new(emqx_ctl, [passthrough, no_history, no_link]), + ok = meck:expect(emqx_ctl, usage, + fun(L) -> emqx_ctl:format_usage(L) end), + ok = meck:expect(emqx_ctl, print, + fun(Fmt) -> + Self ! {fmt, emqx_ctl:format(Fmt)} + end), + ok = meck:expect(emqx_ctl, print, + fun(Fmt, Args) -> + Self ! {fmt, emqx_ctl:format(Fmt, Args)} + end), + Conf. + +end_per_testcase(_, _) -> + meck:unload([emqx_ctl]), + ok. + +%%-------------------------------------------------------------------- +%% Cases +%%-------------------------------------------------------------------- + +%% TODO: + +t_load_unload(_) -> + ok. + +t_gateway_registry_usage(_) -> + ?assertEqual( + ["gateway-registry list # List all registered gateways\n"], + emqx_gateway_cli:'gateway-registry'(usage)). + +t_gateway_registry_list(_) -> + emqx_gateway_cli:'gateway-registry'(["list"]), + ?assertEqual( + "Registered Name: coap, Callback Module: emqx_coap_impl\n" + "Registered Name: exproto, Callback Module: emqx_exproto_impl\n" + "Registered Name: lwm2m, Callback Module: emqx_lwm2m_impl\n" + "Registered Name: mqttsn, Callback Module: emqx_sn_impl\n" + "Registered Name: stomp, Callback Module: emqx_stomp_impl\n" + , acc_print()). + +t_gateway_usage(_) -> + ?assertEqual( + ["gateway list # List all gateway\n", + "gateway lookup # Lookup a gateway detailed informations\n", + "gateway load # Load a gateway with config\n", + "gateway unload # Unload the gateway\n", + "gateway stop # Stop the gateway\n", + "gateway start # Start the gateway\n"], + emqx_gateway_cli:gateway(usage) + ). + +t_gateway_list(_) -> + emqx_gateway_cli:gateway(["list"]), + ?assertEqual( + "Gateway(name=coap, status=unloaded)\n" + "Gateway(name=exproto, status=unloaded)\n" + "Gateway(name=lwm2m, status=unloaded)\n" + "Gateway(name=mqttsn, status=unloaded)\n" + "Gateway(name=stomp, status=unloaded)\n" + , acc_print()). + +t_gateway_load(_) -> + ok. + +t_gateway_unload(_) -> + ok. + +t_gateway_start(_) -> + ok. + +t_gateway_stop(_) -> + ok. + +t_gateway_clients_usage(_) -> + ok. + +t_gateway_clients_list(_) -> + ok. + +t_gateway_clients_lookup(_) -> + ok. + +t_gateway_clients_kick(_) -> + ok. + +t_gateway_metrcis_usage(_) -> + ok. + +t_gateway_metrcis(_) -> + ok. + +acc_print() -> + lists:concat(lists:reverse(acc_print([]))). + +acc_print(Acc) -> + receive + {fmt, S} -> acc_print([S|Acc]) + after 200 -> + Acc + end. From 76b35910b7a0272785a1bc630e897ada3db10fd9 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 27 Dec 2021 09:15:21 +0800 Subject: [PATCH 12/99] chore(gw): fix elvis warnings --- apps/emqx_gateway/src/emqx_gateway_cli.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl index 030c0057b..cc0e09a40 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cli.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -28,6 +28,8 @@ %, 'gateway-banned'/1 ]). +-elvis([{elvis_style, function_naming_convention, disable}]). + -spec load() -> ok. load() -> Cmds = [Fun || {Fun, _} <- ?MODULE:module_info(exports), is_cmd(Fun)], From e9c2b5d1cf55c7a269a5b638b96938a060acce5c Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 27 Dec 2021 16:17:59 +0800 Subject: [PATCH 13/99] fix(rewrite): don't check and translate body for topic rewrite APIs --- apps/emqx_modules/src/emqx_rewrite_api.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_modules/src/emqx_rewrite_api.erl b/apps/emqx_modules/src/emqx_rewrite_api.erl index 3f92cd11f..8435385f2 100644 --- a/apps/emqx_modules/src/emqx_rewrite_api.erl +++ b/apps/emqx_modules/src/emqx_rewrite_api.erl @@ -33,7 +33,7 @@ ]). api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}). + emqx_dashboard_swagger:spec(?MODULE). paths() -> ["/mqtt/topic_rewrite"]. From 9b21945892c704f5db6e35a5c61a88ec0d2d253d Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 27 Dec 2021 16:20:41 +0800 Subject: [PATCH 14/99] fix(rewrite): rewrite to utf8 topics failed --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 1894d817d..2bb8b2d8b 100644 --- a/rebar.config +++ b/rebar.config @@ -46,7 +46,7 @@ {deps, [ {lc, {git, "https://github.com/qzhuyan/lc.git", {tag, "0.1.2"}}} , {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps - , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}} + , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.6"}}} , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.12"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} From 8203b1f3286f6741c0bf937b9a602ecb4460f7f9 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 27 Dec 2021 15:26:45 +0800 Subject: [PATCH 15/99] refactor(gw): simplify massive repeated codes --- apps/emqx_gateway/src/coap/emqx_coap_impl.erl | 93 ++------ .../src/emqx_gateway_api_listeners.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_schema.erl | 2 + apps/emqx_gateway/src/emqx_gateway_utils.erl | 133 ++++++++++- .../src/exproto/emqx_exproto_impl.erl | 208 +++++++----------- .../src/lwm2m/emqx_lwm2m_impl.erl | 93 ++------ apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl | 96 +++----- .../src/stomp/emqx_stomp_impl.erl | 104 +++------ 8 files changed, 301 insertions(+), 430 deletions(-) diff --git a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl index 8bdf1bb3c..49cc3322c 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl @@ -16,9 +16,16 @@ -module(emqx_coap_impl). +-behaviour(emqx_gateway_impl). + +-include_lib("emqx/include/logger.hrl"). -include_lib("emqx_gateway/include/emqx_gateway.hrl"). --behaviour(emqx_gateway_impl). +-import(emqx_gateway_utils, + [ normalize_config/1 + , start_listeners/4 + , stop_listeners/2 + ]). %% APIs -export([ reg/0 @@ -30,8 +37,6 @@ , on_gateway_unload/2 ]). --include_lib("emqx/include/logger.hrl"). - %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -51,12 +56,20 @@ unreg() -> on_gateway_load(_Gateway = #{name := GwName, config := Config }, Ctx) -> - Listeners = emqx_gateway_utils:normalize_config(Config), - ListenerPids = lists:map(fun(Lis) -> - start_listener(GwName, Ctx, Lis) - end, Listeners), - - {ok, ListenerPids, #{ctx => Ctx}}. + Listeners = normalize_config(Config), + ModCfg = #{frame_mod => emqx_coap_frame, + chann_mod => emqx_coap_channel + }, + case start_listeners( + Listeners, GwName, Ctx, ModCfg) of + {ok, ListenerPids} -> + {ok, ListenerPids, #{ctx => Ctx}}; + {error, {Reason, Listener}} -> + throw({badconf, #{ key => listeners + , vallue => Listener + , reason => Reason + }}) + end. on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> GwName = maps:get(name, Gateway), @@ -76,63 +89,5 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> on_gateway_unload(_Gateway = #{ name := GwName, config := Config }, _GwState) -> - Listeners = emqx_gateway_utils:normalize_config(Config), - lists:foreach(fun(Lis) -> - stop_listener(GwName, Lis) - end, Listeners). - -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- - -start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> - ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), - case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of - {ok, Pid} -> - console_print("Gateway ~ts:~ts:~ts on ~ts started.~n", - [GwName, Type, LisName, ListenOnStr]), - Pid; - {error, Reason} -> - ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", - [GwName, Type, LisName, ListenOnStr, Reason]), - throw({badconf, Reason}) - end. - -start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> - Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), - NCfg = Cfg#{ctx => Ctx, - listener => {GwName, Type, LisName}, - frame_mod => emqx_coap_frame, - chann_mod => emqx_coap_channel - }, - MFA = {emqx_gateway_conn, start_link, [NCfg]}, - do_start_listener(Type, Name, ListenOn, SocketOpts, MFA). - -do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) -> - esockd:open_udp(Name, ListenOn, SocketOpts, MFA); - -do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) -> - esockd:open_dtls(Name, ListenOn, SocketOpts, MFA). - -stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> - StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), - ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), - case StopRet of - ok -> - console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n", - [GwName, Type, LisName, ListenOnStr]); - {error, Reason} -> - ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", - [GwName, Type, LisName, ListenOnStr, Reason]) - end, - StopRet. - -stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> - Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), - esockd:close(Name, ListenOn). - --ifndef(TEST). -console_print(Fmt, Args) -> ?ULOG(Fmt, Args). --else. -console_print(_Fmt, _Args) -> ok. --endif. + Listeners = normalize_config(Config), + stop_listeners(GwName, Listeners). diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index ad381ce44..34187224e 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -580,7 +580,7 @@ common_listener_opts() -> #{ nullable => {true, recursively} , desc => <<"The authenticatior for this listener">> })} - ]. + ] ++ emqx_gateway_schema:proxy_protocol_opts(). %%-------------------------------------------------------------------- %% examples diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index cc14eaa33..294e32375 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -50,6 +50,8 @@ -export([namespace/0, roots/0 , fields/1]). +-export([proxy_protocol_opts/0]). + namespace() -> gateway. roots() -> [gateway]. diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 8a81584d6..3fd3045cb 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -18,6 +18,7 @@ -module(emqx_gateway_utils). -include("emqx_gateway.hrl"). +-include_lib("emqx/include/logger.hrl"). -export([ childspec/2 , childspec/3 @@ -26,6 +27,12 @@ , find_sup_child/2 ]). +-export([ start_listeners/4 + , start_listener/4 + , stop_listeners/2 + , stop_listener/2 + ]). + -export([ apply/2 , format_listenon/1 , parse_listenon/1 @@ -89,9 +96,15 @@ childspec(Id, Type, Mod, Args) -> -spec supervisor_ret(supervisor:startchild_ret()) -> {ok, pid()} | {error, supervisor:startchild_err()}. -supervisor_ret({ok, Pid, _Info}) -> {ok, Pid}; -supervisor_ret({error, {Reason, _Child}}) -> {error, Reason}; -supervisor_ret(Ret) -> Ret. +supervisor_ret({ok, Pid, _Info}) -> + {ok, Pid}; +supervisor_ret({error, {Reason, Child}}) -> + case element(1, Child) == child of + true -> {error, Reason}; + _ -> {error, {Reason, Child}} + end; +supervisor_ret(Ret) -> + Ret. -spec find_sup_child(Sup :: pid() | atom(), ChildId :: supervisor:child_id()) -> false @@ -102,6 +115,120 @@ find_sup_child(Sup, ChildId) -> {_Id, Pid, _Type, _Mods} -> {ok, Pid} end. +%% @doc start listeners. close all listeners if someone failed +-spec start_listeners(Listeners :: list(), + GwName :: atom(), + Ctx :: map(), + ModCfg) + -> {ok, [pid()]} + | {error, term()} + when ModCfg :: #{frame_mod := atom(), chann_mod := atom()}. +start_listeners(Listeners, GwName, Ctx, ModCfg) -> + start_listeners(Listeners, GwName, Ctx, ModCfg, []). + +start_listeners([], _, _, _, Acc) -> + {ok, lists:map(fun({listener, {_, Pid}}) -> Pid end, Acc)}; +start_listeners([L | Ls], GwName, Ctx, ModCfg, Acc) -> + case start_listener(GwName, Ctx, L, ModCfg) of + {ok, {ListenerId, ListenOn, Pid}} -> + NAcc = Acc ++ [{listener, {{ListenerId, ListenOn}, Pid}}], + start_listeners(Ls, GwName, Ctx, ModCfg, NAcc); + {error, Reason} -> + lists:foreach(fun({listener, {{ListenerId, ListenOn}, _}}) -> + esockd:close({ListenerId, ListenOn}) + end, Acc), + {error, {Reason, L}} + end. + +-spec start_listener(GwName :: atom(), + Ctx :: emqx_gateway_ctx:context(), + Listener :: tuple(), + ModCfg :: map()) + -> {ok, pid()} + | {error, term()}. +start_listener(GwName, Ctx, + {Type, LisName, ListenOn, SocketOpts, Cfg}, ModCfg) -> + ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), + ListenerId = emqx_gateway_utils:listener_id(GwName, Type, LisName), + + NCfg = maps:merge(Cfg, ModCfg), + case start_listener(GwName, Ctx, Type, + LisName, ListenOn, SocketOpts, NCfg) of + {ok, Pid} -> + console_print("Gateway ~ts:~ts:~ts on ~ts started.~n", + [GwName, Type, LisName, ListenOnStr]), + {ok, {ListenerId, ListenOn, Pid}}; + {error, Reason} -> + ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", + [GwName, Type, LisName, ListenOnStr, Reason]), + emqx_gateway_utils:supervisor_ret({error, Reason}) + end. + +start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), + NCfg = Cfg#{ ctx => Ctx + , listener => {GwName, Type, LisName} + }, + NSocketOpts = merge_default(Type, SocketOpts), + MFA = {emqx_gateway_conn, start_link, [NCfg]}, + do_start_listener(Type, Name, ListenOn, NSocketOpts, MFA). + +merge_default(Udp, Options) -> + {Key, Default} = case Udp of + udp -> + {udp_options, default_udp_options()}; + dtls -> + {udp_options, default_udp_options()}; + tcp -> + {tcp_options, default_tcp_options()}; + ssl -> + {tcp_options, default_tcp_options()} + end, + case lists:keytake(Key, 1, Options) of + {value, {Key, TcpOpts}, Options1} -> + [{Key, emqx_misc:merge_opts(Default, TcpOpts)} + | Options1]; + false -> + [{Key, Default} | Options] + end. + +do_start_listener(Type, Name, ListenOn, SocketOpts, MFA) + when Type == tcp; + Type == ssl -> + esockd:open(Name, ListenOn, SocketOpts, MFA); +do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) -> + esockd:open_udp(Name, ListenOn, SocketOpts, MFA); +do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) -> + esockd:open_dtls(Name, ListenOn, SocketOpts, MFA). + +-spec stop_listeners(GwName :: atom(), Listeners :: list()) -> ok. +stop_listeners(GwName, Listeners) -> + lists:foreach(fun(L) -> stop_listener(GwName, L) end, Listeners). + +-spec stop_listener(GwName :: atom(), Listener :: tuple()) -> ok. +stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> + StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), + ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), + case StopRet of + ok -> + console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n", + [GwName, Type, LisName, ListenOnStr]); + {error, Reason} -> + ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", + [GwName, Type, LisName, ListenOnStr, Reason]) + end, + StopRet. + +stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), + esockd:close(Name, ListenOn). + +-ifndef(TEST). +console_print(Fmt, Args) -> ?ULOG(Fmt, Args). +-else. +console_print(_Fmt, _Args) -> ok. +-endif. + apply({M, F, A}, A2) when is_atom(M), is_atom(M), is_list(A), diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl index 46e3a1628..48dca4324 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl @@ -19,6 +19,14 @@ -behaviour(emqx_gateway_impl). +-include_lib("emqx/include/logger.hrl"). + +-import(emqx_gateway_utils, + [ normalize_config/1 + , start_listeners/4 + , stop_listeners/2 + ]). + %% APIs -export([ reg/0 , unreg/0 @@ -29,8 +37,6 @@ , on_gateway_unload/2 ]). --include_lib("emqx/include/logger.hrl"). - %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -47,6 +53,73 @@ unreg() -> %% emqx_gateway_registry callbacks %%-------------------------------------------------------------------- +on_gateway_load(_Gateway = #{ name := GwName, + config := Config + }, Ctx) -> + %% XXX: How to monitor it ? + %% Start grpc client pool & client channel + PoolName = pool_name(GwName), + PoolSize = emqx_vm:schedulers() * 2, + {ok, PoolSup} = emqx_pool_sup:start_link( + PoolName, hash, PoolSize, + {emqx_exproto_gcli, start_link, []}), + _ = start_grpc_client_channel(GwName, + maps:get(handler, Config, undefined) + ), + %% XXX: How to monitor it ? + _ = start_grpc_server(GwName, maps:get(server, Config, undefined)), + + NConfig = maps:without( + [server, handler], + Config#{pool_name => PoolName} + ), + Listeners = emqx_gateway_utils:normalize_config( + NConfig#{handler => GwName} + ), + + ModCfg = #{frame_mod => emqx_exproto_frame, + chann_mod => emqx_exproto_channel + }, + case start_listeners( + Listeners, GwName, Ctx, ModCfg) of + {ok, ListenerPids} -> + {ok, ListenerPids, _GwState = #{ctx => Ctx, pool => PoolSup}}; + {error, {Reason, Listener}} -> + throw({badconf, #{ key => listeners + , vallue => Listener + , reason => Reason + }}) + end. + +on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> + GwName = maps:get(name, Gateway), + try + %% XXX: 1. How hot-upgrade the changes ??? + %% XXX: 2. Check the New confs first before destroy old instance ??? + on_gateway_unload(Gateway, GwState), + on_gateway_load(Gateway#{config => Config}, Ctx) + catch + Class : Reason : Stk -> + logger:error("Failed to update ~ts; " + "reason: {~0p, ~0p} stacktrace: ~0p", + [GwName, Class, Reason, Stk]), + {error, {Class, Reason}} + end. + +on_gateway_unload(_Gateway = #{ name := GwName, + config := Config + }, _GwState = #{pool := PoolSup}) -> + Listeners = emqx_gateway_utils:normalize_config(Config), + %% Stop funcs??? + exit(PoolSup, kill), + stop_grpc_server(GwName), + stop_grpc_client_channel(GwName), + stop_listeners(GwName, Listeners). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + start_grpc_server(_GwName, undefined) -> undefined; start_grpc_server(GwName, Options = #{bind := ListenOn}) -> @@ -103,140 +176,9 @@ stop_grpc_client_channel(GwName) -> _ = grpc_client_sup:stop_channel_pool(GwName), ok. -on_gateway_load(_Gateway = #{ name := GwName, - config := Config - }, Ctx) -> - %% XXX: How to monitor it ? - %% Start grpc client pool & client channel - PoolName = pool_name(GwName), - PoolSize = emqx_vm:schedulers() * 2, - {ok, PoolSup} = emqx_pool_sup:start_link( - PoolName, hash, PoolSize, - {emqx_exproto_gcli, start_link, []}), - _ = start_grpc_client_channel(GwName, - maps:get(handler, Config, undefined) - ), - %% XXX: How to monitor it ? - _ = start_grpc_server(GwName, maps:get(server, Config, undefined)), - - NConfig = maps:without( - [server, handler], - Config#{pool_name => PoolName} - ), - Listeners = emqx_gateway_utils:normalize_config( - NConfig#{handler => GwName} - ), - ListenerPids = lists:map(fun(Lis) -> - start_listener(GwName, Ctx, Lis) - end, Listeners), - {ok, ListenerPids, _GwState = #{ctx => Ctx, pool => PoolSup}}. - -on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> - GwName = maps:get(name, Gateway), - try - %% XXX: 1. How hot-upgrade the changes ??? - %% XXX: 2. Check the New confs first before destroy old instance ??? - on_gateway_unload(Gateway, GwState), - on_gateway_load(Gateway#{config => Config}, Ctx) - catch - Class : Reason : Stk -> - logger:error("Failed to update ~ts; " - "reason: {~0p, ~0p} stacktrace: ~0p", - [GwName, Class, Reason, Stk]), - {error, {Class, Reason}} - end. - -on_gateway_unload(_Gateway = #{ name := GwName, - config := Config - }, _GwState = #{pool := PoolSup}) -> - Listeners = emqx_gateway_utils:normalize_config(Config), - %% Stop funcs??? - exit(PoolSup, kill), - stop_grpc_server(GwName), - stop_grpc_client_channel(GwName), - lists:foreach(fun(Lis) -> - stop_listener(GwName, Lis) - end, Listeners). - pool_name(GwName) -> list_to_atom(lists:concat([GwName, "_gcli_pool"])). -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- - -start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> - ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), - case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of - {ok, Pid} -> - console_print("Gateway ~ts:~ts:~ts on ~ts started.~n", - [GwName, Type, LisName, ListenOnStr]), - Pid; - {error, Reason} -> - ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", - [GwName, Type, LisName, ListenOnStr, Reason]), - throw({badconf, Reason}) - end. - -start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> - Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), - NCfg = Cfg#{ - ctx => Ctx, - listener => {GwName, Type, LisName}, - frame_mod => emqx_exproto_frame, - chann_mod => emqx_exproto_channel - }, - MFA = {emqx_gateway_conn, start_link, [NCfg]}, - NSockOpts = merge_default_by_type(Type, SocketOpts), - do_start_listener(Type, Name, ListenOn, NSockOpts, MFA). - -do_start_listener(Type, Name, ListenOn, Opts, MFA) - when Type == tcp; - Type == ssl -> - esockd:open(Name, ListenOn, Opts, MFA); -do_start_listener(udp, Name, ListenOn, Opts, MFA) -> - esockd:open_udp(Name, ListenOn, Opts, MFA); -do_start_listener(dtls, Name, ListenOn, Opts, MFA) -> - esockd:open_dtls(Name, ListenOn, Opts, MFA). - -merge_default_by_type(Type, Options) when Type =:= tcp; - Type =:= ssl -> - Default = emqx_gateway_utils:default_tcp_options(), - case lists:keytake(tcp_options, 1, Options) of - {value, {tcp_options, TcpOpts}, Options1} -> - [{tcp_options, emqx_misc:merge_opts(Default, TcpOpts)} - | Options1]; - false -> - [{tcp_options, Default} | Options] - end; -merge_default_by_type(Type, Options) when Type =:= udp; - Type =:= dtls -> - Default = emqx_gateway_utils:default_udp_options(), - case lists:keytake(udp_options, 1, Options) of - {value, {udp_options, TcpOpts}, Options1} -> - [{udp_options, emqx_misc:merge_opts(Default, TcpOpts)} - | Options1]; - false -> - [{udp_options, Default} | Options] - end. - -stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> - StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), - ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), - case StopRet of - ok -> - console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n", - [GwName, Type, LisName, ListenOnStr]); - {error, Reason} -> - ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", - [GwName, Type, LisName, ListenOnStr, Reason]) - end, - StopRet. - -stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> - Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), - esockd:close(Name, ListenOn). - -ifndef(TEST). console_print(Fmt, Args) -> ?ULOG(Fmt, Args). -else. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl index ee27d89b1..edf035240 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl @@ -19,6 +19,8 @@ -behaviour(emqx_gateway_impl). +-include_lib("emqx/include/logger.hrl"). + %% APIs -export([ reg/0 , unreg/0 @@ -29,8 +31,6 @@ , on_gateway_unload/2 ]). --include_lib("emqx/include/logger.hrl"). - %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -54,10 +54,20 @@ on_gateway_load(_Gateway = #{ name := GwName, case emqx_lwm2m_xml_object_db:start_link(XmlDir) of {ok, RegPid} -> Listeners = emqx_gateway_utils:normalize_config(Config), - ListenerPids = lists:map(fun(Lis) -> - start_listener(GwName, Ctx, Lis) - end, Listeners), - {ok, ListenerPids, _GwState = #{ctx => Ctx, registry => RegPid}}; + ModCfg = #{frame_mod => emqx_coap_frame, + chann_mod => emqx_lwm2m_channel + }, + case emqx_gateway_utils:start_listeners( + Listeners, GwName, Ctx, ModCfg) of + {ok, ListenerPids} -> + {ok, ListenerPids, #{ctx => Ctx, registry => RegPid}}; + {error, {Reason, Listener}} -> + emqx_lwm2m_xml_object_db:stop(), + throw({badconf, #{ key => listeners + , vallue => Listener + , reason => Reason + }}) + end; {error, Reason} -> throw({badconf, #{ key => xml_dir , value => XmlDir @@ -85,73 +95,4 @@ on_gateway_unload(_Gateway = #{ name := GwName, }, _GwState = #{registry := RegPid}) -> exit(RegPid, kill), Listeners = emqx_gateway_utils:normalize_config(Config), - lists:foreach(fun(Lis) -> - stop_listener(GwName, Lis) - end, Listeners). - -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- - -start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> - ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), - case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of - {ok, Pid} -> - console_print("Gateway ~ts:~ts:~ts on ~ts started.~n", - [GwName, Type, LisName, ListenOnStr]), - Pid; - {error, Reason} -> - ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", - [GwName, Type, LisName, ListenOnStr, Reason]), - throw({badconf, Reason}) - end. - -start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> - Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), - NCfg = Cfg#{ ctx => Ctx - , listener => {GwName, Type, LisName} - , frame_mod => emqx_coap_frame - , chann_mod => emqx_lwm2m_channel - }, - NSocketOpts = merge_default(SocketOpts), - MFA = {emqx_gateway_conn, start_link, [NCfg]}, - do_start_listener(Type, Name, ListenOn, NSocketOpts, MFA). - -merge_default(Options) -> - Default = emqx_gateway_utils:default_udp_options(), - case lists:keytake(udp_options, 1, Options) of - {value, {udp_options, TcpOpts}, Options1} -> - [{udp_options, emqx_misc:merge_opts(Default, TcpOpts)} - | Options1]; - false -> - [{udp_options, Default} | Options] - end. - -do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) -> - esockd:open_udp(Name, ListenOn, SocketOpts, MFA); - -do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) -> - esockd:open_dtls(Name, ListenOn, SocketOpts, MFA). - -stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> - StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), - ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), - case StopRet of - ok -> - console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n", - [GwName, Type, LisName, ListenOnStr]); - {error, Reason} -> - ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", - [GwName, Type, LisName, ListenOnStr, Reason]) - end, - StopRet. - -stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> - Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), - esockd:close(Name, ListenOn). - --ifndef(TEST). -console_print(Fmt, Args) -> ?ULOG(Fmt, Args). --else. -console_print(_Fmt, _Args) -> ok. --endif. + emqx_gateway_utils:stop_listeners(GwName, Listeners). diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl index 377c4f6d6..4284af626 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl @@ -19,6 +19,14 @@ -behaviour(emqx_gateway_impl). +-include_lib("emqx/include/logger.hrl"). + +-import(emqx_gateway_utils, + [ normalize_config/1 + , start_listeners/4 + , stop_listeners/2 + ]). + %% APIs -export([ reg/0 , unreg/0 @@ -29,8 +37,6 @@ , on_gateway_unload/2 ]). --include_lib("emqx/include/logger.hrl"). - %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -70,12 +76,23 @@ on_gateway_load(_Gateway = #{ name := GwName, [broadcast, predefined], Config#{registry => emqx_sn_registry:lookup_name(RegistrySvr)} ), + Listeners = emqx_gateway_utils:normalize_config(NConfig), - ListenerPids = lists:map(fun(Lis) -> - start_listener(GwName, Ctx, Lis) - end, Listeners), - {ok, ListenerPids, _InstaState = #{ctx => Ctx}}. + ModCfg = #{frame_mod => emqx_sn_frame, + chann_mod => emqx_sn_channel + }, + + case start_listeners( + Listeners, GwName, Ctx, ModCfg) of + {ok, ListenerPids} -> + {ok, ListenerPids, _GwState = #{ctx => Ctx}}; + {error, {Reason, Listener}} -> + throw({badconf, #{ key => listeners + , vallue => Listener + , reason => Reason + }}) + end. on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> GwName = maps:get(name, Gateway), @@ -95,68 +112,5 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> on_gateway_unload(_Gateway = #{ name := GwName, config := Config }, _GwState) -> - Listeners = emqx_gateway_utils:normalize_config(Config), - lists:foreach(fun(Lis) -> - stop_listener(GwName, Lis) - end, Listeners). - -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- - -start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> - ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), - case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of - {ok, Pid} -> - console_print("Gateway ~ts:~ts:~ts on ~ts started.~n", - [GwName, Type, LisName, ListenOnStr]), - Pid; - {error, Reason} -> - ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", - [GwName, Type, LisName, ListenOnStr, Reason]), - throw({badconf, Reason}) - end. - -start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> - Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), - NCfg = Cfg#{ - ctx => Ctx, - listene => {GwName, Type, LisName}, - frame_mod => emqx_sn_frame, - chann_mod => emqx_sn_channel - }, - esockd:open_udp(Name, ListenOn, merge_default(SocketOpts), - {emqx_gateway_conn, start_link, [NCfg]}). - -merge_default(Options) -> - Default = emqx_gateway_utils:default_udp_options(), - case lists:keytake(udp_options, 1, Options) of - {value, {udp_options, TcpOpts}, Options1} -> - [{udp_options, emqx_misc:merge_opts(Default, TcpOpts)} - | Options1]; - false -> - [{udp_options, Default} | Options] - end. - -stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> - StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), - ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), - case StopRet of - ok -> - console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n", - [GwName, Type, LisName, ListenOnStr]); - {error, Reason} -> - ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", - [GwName, Type, LisName, ListenOnStr, Reason]) - end, - StopRet. - -stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> - Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), - esockd:close(Name, ListenOn). - --ifndef(TEST). -console_print(Fmt, Args) -> ?ULOG(Fmt, Args). --else. -console_print(_Fmt, _Args) -> ok. --endif. + Listeners = normalize_config(Config), + stop_listeners(GwName, Listeners). diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl index 41df189bc..4e490e181 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl @@ -18,6 +18,15 @@ -behaviour(emqx_gateway_impl). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/include/emqx_gateway.hrl"). + +-import(emqx_gateway_utils, + [ normalize_config/1 + , start_listeners/4 + , stop_listeners/2 + ]). + %% APIs -export([ reg/0 , unreg/0 @@ -28,9 +37,6 @@ , on_gateway_unload/2 ]). --include_lib("emqx_gateway/include/emqx_gateway.hrl"). --include_lib("emqx/include/logger.hrl"). - %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -52,15 +58,22 @@ unreg() -> on_gateway_load(_Gateway = #{ name := GwName, config := Config }, Ctx) -> - %% Step1. Fold the config to listeners - Listeners = emqx_gateway_utils:normalize_config(Config), - %% Step2. Start listeners or escokd:specs - ListenerPids = lists:map(fun(Lis) -> - start_listener(GwName, Ctx, Lis) - end, Listeners), - %% FIXME: How to throw an exception to interrupt the restart logic ? - %% FIXME: Assign ctx to GwState - {ok, ListenerPids, _GwState = #{ctx => Ctx}}. + Listeners = normalize_config(Config), + ModCfg = #{frame_mod => emqx_stomp_frame, + chann_mod => emqx_stomp_channel + }, + case start_listeners( + Listeners, GwName, Ctx, ModCfg) of + {ok, ListenerPids} -> + %% FIXME: How to throw an exception to interrupt the restart logic ? + %% FIXME: Assign ctx to GwState + {ok, ListenerPids, _GwState = #{ctx => Ctx}}; + {error, {Reason, Listener}} -> + throw({badconf, #{ key => listeners + , vallue => Listener + , reason => Reason + }}) + end. on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> GwName = maps:get(name, Gateway), @@ -80,68 +93,5 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> on_gateway_unload(_Gateway = #{ name := GwName, config := Config }, _GwState) -> - Listeners = emqx_gateway_utils:normalize_config(Config), - lists:foreach(fun(Lis) -> - stop_listener(GwName, Lis) - end, Listeners). - -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- - -start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> - ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), - case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of - {ok, Pid} -> - console_print("Gateway ~ts:~ts:~ts on ~ts started.~n", - [GwName, Type, LisName, ListenOnStr]), - Pid; - {error, Reason} -> - ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", - [GwName, Type, LisName, ListenOnStr, Reason]), - throw({badconf, Reason}) - end. - -start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> - Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), - NCfg = Cfg#{ - ctx => Ctx, - listener => {GwName, Type, LisName}, %% Used for authn - frame_mod => emqx_stomp_frame, - chann_mod => emqx_stomp_channel - }, - esockd:open(Name, ListenOn, merge_default(SocketOpts), - {emqx_gateway_conn, start_link, [NCfg]}). - -merge_default(Options) -> - Default = emqx_gateway_utils:default_tcp_options(), - case lists:keytake(tcp_options, 1, Options) of - {value, {tcp_options, TcpOpts}, Options1} -> - [{tcp_options, emqx_misc:merge_opts(Default, TcpOpts)} - | Options1]; - false -> - [{tcp_options, Default} | Options] - end. - -stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> - StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), - ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), - case StopRet of - ok -> - console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n", - [GwName, Type, LisName, ListenOnStr]); - {error, Reason} -> - ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", - [GwName, Type, LisName, ListenOnStr, Reason]) - end, - StopRet. - -stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> - Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), - esockd:close(Name, ListenOn). - --ifndef(TEST). -console_print(Fmt, Args) -> ?ULOG(Fmt, Args). --else. -console_print(_Fmt, _Args) -> ok. --endif. + Listeners = normalize_config(Config), + stop_listeners(GwName, Listeners). From fc89fb0f8a09320b16f4bd71728ad77575340ee2 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 28 Dec 2021 09:16:00 +0800 Subject: [PATCH 16/99] chore: using prepared query in postgresql connector --- apps/emqx_connector/rebar.config | 2 +- apps/emqx_connector/src/emqx_connector_pgsql.erl | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 58706e950..eadc4e773 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -6,7 +6,7 @@ {deps, [ {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}}, {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}}, - {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.6.0"}}}, + {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7-emqx.1"}}}, %% NOTE: mind poolboy version when updating mongodb-erlang version {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.10"}}}, %% NOTE: mind poolboy version when updating eredis_cluster version diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index f42bed666..21dc8d948 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -32,7 +32,7 @@ -export([connect/1]). --export([query/3]). +-export([query/4]). -export([do_health_check/1]). @@ -85,7 +85,7 @@ on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) -> on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) -> ?SLOG(debug, #{msg => "postgresql connector received sql query", connector => InstId, sql => SQL, state => State}), - case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [SQL, Params]}, no_handover) of + case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [InstId, SQL, Params]}, no_handover) of {error, Reason} -> ?SLOG(error, #{ msg => "postgresql connector do sql query failed", @@ -112,8 +112,8 @@ connect(Opts) -> Password = proplists:get_value(password, Opts), epgsql:connect(Host, Username, Password, conn_opts(Opts)). -query(Conn, SQL, Params) -> - epgsql:equery(Conn, SQL, Params). +query(Conn, Name, SQL, Params) -> + epgsql:prepared_query2(Conn, Name, SQL, Params). conn_opts(Opts) -> conn_opts(Opts, []). From d26042703efd24743df2b6486a02c2c2691c3b8e Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 28 Dec 2021 09:52:31 +0800 Subject: [PATCH 17/99] chore: provide prepared_query and equery --- .../src/simple_authn/emqx_authn_pgsql.erl | 2 +- .../test/emqx_authn_pgsql_SUITE.erl | 4 ++-- apps/emqx_authz/src/emqx_authz_postgresql.erl | 2 +- .../test/emqx_authz_postgresql_SUITE.erl | 4 ++-- .../src/emqx_connector_pgsql.erl | 24 ++++++++++++------- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 0ed7d282a..816eace0d 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -106,7 +106,7 @@ authenticate(#{password := Password} = Credential, resource_id := ResourceId, password_hash_algorithm := Algorithm}) -> Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential), - case emqx_resource:query(ResourceId, {sql, Query, Params}) of + case emqx_resource:query(ResourceId, {prepared_query, ResourceId, Query, Params}) of {ok, _Columns, []} -> ignore; {ok, Columns, [Row | _]} -> NColumns = [Name || #column{name = Name} <- Columns], diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl index 8f1f12690..e33f5c100 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl @@ -441,12 +441,12 @@ create_user(Values) -> q(Sql) -> emqx_resource:query( ?PGSQL_RESOURCE, - {sql, Sql}). + {query, Sql}). q(Sql, Params) -> emqx_resource:query( ?PGSQL_RESOURCE, - {sql, Sql, Params}). + {query, Sql, Params}). drop_seeds() -> {ok, _, _} = q("DROP TABLE IF EXISTS users"), diff --git a/apps/emqx_authz/src/emqx_authz_postgresql.erl b/apps/emqx_authz/src/emqx_authz_postgresql.erl index f101841c2..ae57a0ae4 100644 --- a/apps/emqx_authz/src/emqx_authz_postgresql.erl +++ b/apps/emqx_authz/src/emqx_authz_postgresql.erl @@ -73,7 +73,7 @@ authorize(Client, PubSub, Topic, query := {Query, Params} } }) -> - case emqx_resource:query(ResourceID, {sql, Query, replvar(Params, Client)}) of + case emqx_resource:query(ResourceID, {prepared_query, ResourceID, Query, replvar(Params, Client)}) of {ok, _Columns, []} -> nomatch; {ok, Columns, Rows} -> do_authorize(Client, PubSub, Topic, Columns, Rows); diff --git a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl index 92c479f92..a264d3407 100644 --- a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl @@ -228,12 +228,12 @@ raw_pgsql_authz_config() -> q(Sql) -> emqx_resource:query( ?PGSQL_RESOURCE, - {sql, Sql}). + {query, Sql}). insert(Sql, Params) -> {ok, _} = emqx_resource:query( ?PGSQL_RESOURCE, - {sql, Sql, Params}), + {query, Sql, Params}), ok. init_table() -> diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 21dc8d948..450b5cbca 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -32,7 +32,9 @@ -export([connect/1]). --export([query/4]). +-export([ query/3 + , query/4 + ]). -export([do_health_check/1]). @@ -60,8 +62,7 @@ on_start(InstId, #{server := {Host, Port}, connector => InstId, config => Config}), SslOpts = case maps:get(enable, SSL) of true -> - [{ssl, [{server_name_indication, disable} | - emqx_plugin_libs_ssl:save_files_return_opts(SSL, "connectors", InstId)]}]; + [{ssl, [emqx_plugin_libs_ssl:save_files_return_opts(SSL, "connectors", InstId)]}]; false -> [] end, Options = [{host, Host}, @@ -80,12 +81,16 @@ on_stop(InstId, #{poolname := PoolName}) -> connector => InstId}), emqx_plugin_libs_pool:stop_pool(PoolName). -on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) -> - on_query(InstId, {sql, SQL, []}, AfterQuery, State); -on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) -> +on_query(InstId, QueryParams, AfterQuery, #{poolname := PoolName} = State) -> + {Command, Args} = case QueryParams of + {query, SQL} -> {query, [SQL, []]}; + {query, SQL, Params} -> {query, [SQL, Params]}; + {prepared_query, Name, SQL} -> {prepared_query, [Name, SQL, []]}; + {prepared_query, Name, SQL, Params} -> {prepared_query, [Name, SQL, Params]} + end, ?SLOG(debug, #{msg => "postgresql connector received sql query", - connector => InstId, sql => SQL, state => State}), - case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [InstId, SQL, Params]}, no_handover) of + connector => InstId, command => Command, args => Args, state => State}), + case Result = ecpool:pick_and_do(PoolName, {?MODULE, Command, Args}, no_handover) of {error, Reason} -> ?SLOG(error, #{ msg => "postgresql connector do sql query failed", @@ -112,6 +117,9 @@ connect(Opts) -> Password = proplists:get_value(password, Opts), epgsql:connect(Host, Username, Password, conn_opts(Opts)). +query(Conn, SQL, Params) -> + epgsql:equery(Conn, SQL, Params). + query(Conn, Name, SQL, Params) -> epgsql:prepared_query2(Conn, Name, SQL, Params). From 1cd226c18bd756dc569dda7a37530b89ad2b8ed9 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 28 Dec 2021 10:12:39 +0800 Subject: [PATCH 18/99] fix(bridge): make direction defaults to egress if not provided --- apps/emqx_bridge/src/emqx_bridge_http_schema.erl | 1 + apps/emqx_bridge/src/emqx_bridge_schema.erl | 1 + 2 files changed, 2 insertions(+) diff --git a/apps/emqx_bridge/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge/src/emqx_bridge_http_schema.erl index 43cace332..540a6a070 100644 --- a/apps/emqx_bridge/src/emqx_bridge_http_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_http_schema.erl @@ -87,6 +87,7 @@ basic_config() -> , {direction, mk(egress, #{ desc => "The direction of this bridge, MUST be egress" + , default => egress })} ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)). diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl index 3acfbcdef..82fc79ebf 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -71,6 +71,7 @@ metrics_status_fields() -> direction_field(Dir, Desc) -> {direction, mk(Dir, #{ nullable => false + , default => egress , desc => "The direction of the bridge. Can be one of 'ingress' or 'egress'.
" ++ Desc })}. From c68499e39a231a4831ed25cf3133ef26c6bb9c39 Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 24 Dec 2021 14:52:10 +0800 Subject: [PATCH 19/99] fix(api): fix the payload in the result of the api of emqx_retainer & emqx_dealy --- apps/emqx_modules/src/emqx_delayed_api.erl | 7 +-- apps/emqx_retainer/src/emqx_retainer_api.erl | 51 +++++++++----------- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/apps/emqx_modules/src/emqx_delayed_api.erl b/apps/emqx_modules/src/emqx_delayed_api.erl index 8137d9e63..9199d7b2c 100644 --- a/apps/emqx_modules/src/emqx_delayed_api.erl +++ b/apps/emqx_modules/src/emqx_delayed_api.erl @@ -44,6 +44,7 @@ -define(MESSAGE_ID_NOT_FOUND, 'MESSAGE_ID_NOT_FOUND'). -define(MESSAGE_ID_SCHEMA_ERROR, 'MESSAGE_ID_SCHEMA_ERROR'). +-define(MAX_PAYLOAD_SIZE, 1048576). %% 1MB = 1024 x 1024 api_spec() -> emqx_dashboard_swagger:spec(?MODULE). @@ -157,11 +158,11 @@ delayed_message(get, #{bindings := #{msgid := Id}}) -> case emqx_delayed:get_delayed_message(Id) of {ok, Message} -> Payload = maps:get(payload, Message), - case size(Payload) > ?MAX_PAYLOAD_LENGTH of + case erlang:byte_size(Payload) > ?MAX_PAYLOAD_SIZE of true -> - {200, Message#{payload => ?PAYLOAD_TOO_LARGE}}; + {200, Message}; _ -> - {200, Message#{payload => Payload}} + {200, Message#{payload => base64:encode(Payload)}} end; {error, id_schema_error} -> {400, generate_http_code_map(id_schema_error, Id)}; diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index 26d341b53..5424629d9 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -28,13 +28,14 @@ -import(emqx_mgmt_api_configs, [gen_schema/1]). -import(emqx_mgmt_util, [ object_array_schema/2 + , object_schema/2 , schema/1 , schema/2 , error_schema/2 , page_params/0 , properties/1]). --define(MAX_BASE64_PAYLOAD_SIZE, 1048576). %% 1MB = 1024 x 1024 +-define(MAX_PAYLOAD_SIZE, 1048576). %% 1MB = 1024 x 1024 api_spec() -> {[lookup_retained_api(), with_topic_api(), config_api()], []}. @@ -64,7 +65,7 @@ parameters() -> lookup_retained_api() -> Metadata = #{ get => #{ - description => <<"lookup matching messages">>, + description => <<"List retained messages">>, parameters => page_params(), responses => #{ <<"200">> => object_array_schema( @@ -80,9 +81,10 @@ with_topic_api() -> MetaData = #{ get => #{ description => <<"lookup matching messages">>, - parameters => parameters() ++ page_params(), + parameters => parameters(), responses => #{ - <<"200">> => object_array_schema(message_props(), <<"List retained messages">>), + <<"200">> => object_schema(message_props(), <<"List retained messages">>), + <<"404">> => error_schema(<<"Reatined Not Exists">>, ['NOT_FOUND']), <<"405">> => schema(<<"NotAllowed">>) } }, @@ -139,35 +141,27 @@ config(put, #{body := Body}) -> %%------------------------------------------------------------------------------ %% Interval Funcs %%------------------------------------------------------------------------------ -lookup_retained(get, Params) -> - lookup(undefined, Params, fun format_message/1). +lookup_retained(get, #{query_string := Qs}) -> + Page = maps:get(page, Qs, 1), + Limit = maps:get(page, Qs, emqx_mgmt:max_row_limit()), + {ok, Msgs} = emqx_retainer_mnesia:page_read(undefined, undefined, Page, Limit), + {200, [format_message(Msg) || Msg <- Msgs]}. -with_topic(get, #{bindings := Bindings} = Params) -> +with_topic(get, #{bindings := Bindings}) -> Topic = maps:get(topic, Bindings), - lookup(Topic, Params, fun format_detail_message/1); + {ok, Msgs} = emqx_retainer_mnesia:page_read(undefined, Topic, 1, 1), + case Msgs of + [H | _] -> + {200, format_detail_message(H)}; + _ -> + {404, #{code => 'NOT_FOUND'}} + end; with_topic(delete, #{bindings := Bindings}) -> Topic = maps:get(topic, Bindings), emqx_retainer_mnesia:delete_message(undefined, Topic), {204}. --spec lookup(undefined | binary(), - map(), - fun((emqx_types:message()) -> map())) -> - {200, map()}. -lookup(Topic, #{query_string := Qs}, Formatter) -> - Page = maps:get(page, Qs, 1), - Limit = maps:get(page, Qs, emqx_mgmt:max_row_limit()), - {ok, Msgs} = emqx_retainer_mnesia:page_read(undefined, Topic, Page, Limit), - {200, format_message(Msgs, Formatter)}. - - -format_message(Messages, Formatter) when is_list(Messages)-> - [Formatter(Message) || Message <- Messages]; - -format_message(Message, Formatter) -> - Formatter(Message). - format_message(#message{ id = ID, qos = Qos, topic = Topic, from = From , timestamp = Timestamp, headers = Headers}) -> #{msgid => emqx_guid:to_hexstr(ID), @@ -181,12 +175,11 @@ format_message(#message{ id = ID, qos = Qos, topic = Topic, from = From format_detail_message(#message{payload = Payload} = Msg) -> Base = format_message(Msg), - EncodePayload = base64:encode(Payload), - case erlang:byte_size(EncodePayload) =< ?MAX_BASE64_PAYLOAD_SIZE of + case erlang:byte_size(Payload) =< ?MAX_PAYLOAD_SIZE of true -> - Base#{payload => EncodePayload}; + Base#{payload => base64:encode(Payload)}; _ -> - Base#{payload => base64:encode(<<"PAYLOAD_TOO_LARGE">>)} + Base end. to_bin_string(Data) when is_binary(Data) -> From bf2392f682c19ebbc07bbccf9514069c737cc6c6 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 28 Dec 2021 10:52:30 +0800 Subject: [PATCH 20/99] chore(typerefl): update typerefl to 0.8.6 --- apps/emqx/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 330291def..c94208594 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -11,7 +11,7 @@ {deps, [ {lc, {git, "https://github.com/qzhuyan/lc.git", {tag, "0.1.2"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} - , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}} + , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.6"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} From 1a22d5ca46a3abd617ce23726a5763c1d34225d5 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 28 Dec 2021 11:21:41 +0800 Subject: [PATCH 21/99] fix: fix undef function call --- apps/emqx_connector/src/emqx_connector_pgsql.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 450b5cbca..2f09ae59a 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -33,7 +33,7 @@ -export([connect/1]). -export([ query/3 - , query/4 + , prepared_query/4 ]). -export([do_health_check/1]). @@ -120,7 +120,7 @@ connect(Opts) -> query(Conn, SQL, Params) -> epgsql:equery(Conn, SQL, Params). -query(Conn, Name, SQL, Params) -> +prepared_query(Conn, Name, SQL, Params) -> epgsql:prepared_query2(Conn, Name, SQL, Params). conn_opts(Opts) -> From eb992ad2ad15d2b5d70b1b1b4592da342c2f560f Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 28 Dec 2021 11:35:47 +0800 Subject: [PATCH 22/99] fix(bridge): add test cases for sending msgs via http bridge --- .../test/emqx_bridge_api_SUITE.erl | 62 +++++++++++++++---- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 65baf7051..7724d467c 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -47,7 +47,7 @@ groups() -> []. suite() -> - [{timetrap,{seconds,30}}]. + [{timetrap,{seconds,60}}]. init_per_suite(Config) -> ok = emqx_config:put([emqx_dashboard], #{ @@ -84,7 +84,7 @@ start_http_server(HandleFun) -> spawn_link(fun() -> {Port, Sock} = listen_on_random_port(), Parent ! {port, Port}, - loop(Sock, HandleFun) + loop(Sock, HandleFun, Parent) end), receive {port, Port} -> Port @@ -95,40 +95,49 @@ start_http_server(HandleFun) -> listen_on_random_port() -> Min = 1024, Max = 65000, Port = rand:uniform(Max - Min) + Min, - case gen_tcp:listen(Port, [{active, false}, {reuseaddr, true}]) of + case gen_tcp:listen(Port, [{active, false}, {reuseaddr, true}, binary]) of {ok, Sock} -> {Port, Sock}; {error, eaddrinuse} -> listen_on_random_port() end. -loop(Sock, HandleFun) -> +loop(Sock, HandleFun, Parent) -> {ok, Conn} = gen_tcp:accept(Sock), - Handler = spawn(fun () -> HandleFun(Conn) end), + Handler = spawn(fun () -> HandleFun(Conn, Parent) end), gen_tcp:controlling_process(Conn, Handler), - loop(Sock, HandleFun). + loop(Sock, HandleFun, Parent). make_response(CodeStr, Str) -> B = iolist_to_binary(Str), iolist_to_binary( io_lib:fwrite( - "HTTP/1.0 ~s\nContent-Type: text/html\nContent-Length: ~p\n\n~s", + "HTTP/1.0 ~s\r\nContent-Type: text/html\r\nContent-Length: ~p\r\n\r\n~s", [CodeStr, size(B), B])). -handle_fun_200_ok(Conn) -> +handle_fun_200_ok(Conn, Parent) -> case gen_tcp:recv(Conn, 0) of - {ok, Request} -> + {ok, ReqStr} -> + ct:pal("the http handler got request: ~p", [ReqStr]), + Req = parse_http_request(ReqStr), + Parent ! {http_server, received, Req}, gen_tcp:send(Conn, make_response("200 OK", "Request OK")), - self() ! {http_server, received, Request}, - handle_fun_200_ok(Conn); + handle_fun_200_ok(Conn, Parent); {error, closed} -> gen_tcp:close(Conn) end. +parse_http_request(ReqStr0) -> + [Method, ReqStr1] = string:split(ReqStr0, " ", leading), + [Path, ReqStr2] = string:split(ReqStr1, " ", leading), + [_ProtoVsn, ReqStr3] = string:split(ReqStr2, "\r\n", leading), + [_HeaderStr, Body] = string:split(ReqStr3, "\r\n\r\n", leading), + #{method => Method, path => Path, body => Body}. + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ t_http_crud_apis(_) -> - Port = start_http_server(fun handle_fun_200_ok/1), + Port = start_http_server(fun handle_fun_200_ok/2), %% assert we there's no bridges at first {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), @@ -163,6 +172,20 @@ t_http_crud_apis(_) -> , <<"message">> := <<"bridge already exists">> }, jsx:decode(RetMsg)), + %% send an message to emqx and the message should be forwarded to the HTTP server + Body = <<"my msg">>, + emqx:publish(emqx_message:make(<<"emqx_http/1">>, Body)), + ?assert( + receive + {http_server, received, #{method := <<"POST">>, path := <<"/path1">>, + body := Body}} -> + true; + Msg -> + ct:pal("error: http got unexpected request: ~p", [Msg]), + false + after 100 -> + false + end), %% update the request-path of the bridge URL2 = ?URL(Port, "path2"), {ok, 200, Bridge2} = request(put, uri(["bridges", ?BRIDGE_ID]), @@ -201,6 +224,19 @@ t_http_crud_apis(_) -> , <<"url">> := URL2 }, jsx:decode(Bridge3Str)), + %% send an message to emqx again, check the path has been changed + emqx:publish(emqx_message:make(<<"emqx_http/1">>, Body)), + ?assert( + receive + {http_server, received, #{path := <<"/path2">>}} -> + true; + Msg2 -> + ct:pal("error: http got unexpected request: ~p", [Msg2]), + false + after 100 -> + false + end), + %% delete the bridge {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), @@ -215,7 +251,7 @@ t_http_crud_apis(_) -> ok. t_start_stop_bridges(_) -> - Port = start_http_server(fun handle_fun_200_ok/1), + Port = start_http_server(fun handle_fun_200_ok/2), URL1 = ?URL(Port, "abc"), {ok, 201, Bridge} = request(post, uri(["bridges"]), ?HTTP_BRIDGE(URL1)#{ From 487b84f166e77e8a766d808624f071925047f304 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 28 Dec 2021 11:37:51 +0800 Subject: [PATCH 23/99] test: fix test case --- apps/emqx_authz/test/emqx_authz_SUITE.erl | 7 +++---- apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl | 7 +++---- apps/emqx_authz/test/emqx_authz_http_SUITE.erl | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index e18901fc5..7038b59e0 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -31,10 +31,9 @@ groups() -> init_per_suite(Config) -> meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end), - meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end), - meck:expect(emqx_resource, remove, fun(_) -> ok end), - meck:expect(emqx_resource, create_dry_run, fun(_, _) -> ok end), + meck:expect(emqx_resource, create_local, fun(_, _, _) -> {ok, meck_data} end), + meck:expect(emqx_resource, remove_local, fun(_) -> ok end), + meck:expect(emqx_resource, create_dry_run_local, fun(_, _) -> ok end), ok = emqx_common_test_helpers:start_apps( [emqx_connector, emqx_conf, emqx_authz], diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index 67e1b05da..b1b6676ad 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -96,14 +96,13 @@ groups() -> init_per_suite(Config) -> meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end), - meck:expect(emqx_resource, create_dry_run, + meck:expect(emqx_resource, create_local, fun(_, _, _) -> {ok, meck_data} end), + meck:expect(emqx_resource, create_dry_run_local, fun(emqx_connector_mysql, _) -> {ok, meck_data}; (T, C) -> meck:passthrough([T, C]) end), - meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end), meck:expect(emqx_resource, health_check, fun(_) -> ok end), - meck:expect(emqx_resource, remove, fun(_) -> ok end ), + meck:expect(emqx_resource, remove_local, fun(_) -> ok end ), ok = emqx_common_test_helpers:start_apps( [emqx_conf, emqx_authz, emqx_dashboard], diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl index c438b3f4b..939c8fb5e 100644 --- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -30,8 +30,8 @@ groups() -> init_per_suite(Config) -> meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end), - meck:expect(emqx_resource, remove, fun(_) -> ok end ), + meck:expect(emqx_resource, create_local, fun(_, _, _) -> {ok, meck_data} end), + meck:expect(emqx_resource, remove_local, fun(_) -> ok end ), ok = emqx_common_test_helpers:start_apps( [emqx_conf, emqx_authz], From 4406589980fe8d7dd4d886fcc0254f98ed54c57f Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 28 Dec 2021 14:12:28 +0800 Subject: [PATCH 24/99] fix(bridge): time unit error for MQTT keepalive --- apps/emqx_connector/src/emqx_connector_mqtt.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index f8d17ce32..bcf558117 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -208,7 +208,7 @@ basic_config(#{ username => User, password => Password, clean_start => CleanStart, - keepalive => KeepAlive, + keepalive => ms_to_s(KeepAlive), retry_interval => RetryIntv, max_inflight => MaxInflight, ssl => EnableSsl, @@ -216,5 +216,8 @@ basic_config(#{ if_record_metrics => true }. +ms_to_s(Ms) -> + erlang:ceil(Ms / 1000). + clientid(Id) -> iolist_to_binary([Id, ":", atom_to_list(node())]). From 07997ab8652e8b3f709b05197a3294693d1b00b8 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 28 Dec 2021 14:13:26 +0800 Subject: [PATCH 25/99] fix(bridge): Bridges should send a JSON message if payload template not set --- apps/emqx_connector/src/emqx_connector_http.erl | 7 ++++++- apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl | 9 +++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 2b9bd48aa..21c06284d 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -266,11 +266,16 @@ process_request(#{ } = Conf, Msg) -> Conf#{ method => make_method(emqx_plugin_libs_rule:proc_tmpl(MethodTks, Msg)) , path => emqx_plugin_libs_rule:proc_tmpl(PathTks, Msg) - , body => emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg) + , body => process_request_body(BodyTks, Msg) , headers => maps:to_list(proc_headers(HeadersTks, Msg)) , request_timeout => ReqTimeout }. +process_request_body([], Msg) -> + emqx_json:encode(Msg); +process_request_body(BodyTks, Msg) -> + emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg). + proc_headers(HeaderTks, Msg) -> maps:fold(fun(K, V, Acc) -> Acc#{emqx_plugin_libs_rule:proc_tmpl(K, Msg) => diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl index eb483dcc5..1357037ee 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -66,7 +66,7 @@ to_remote_msg(#message{flags = Flags0} = Msg, Vars) -> to_remote_msg(MapMsg, #{remote_topic := TopicToken, payload := PayloadToken, remote_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) when is_map(MapMsg) -> Topic = replace_vars_in_str(TopicToken, MapMsg), - Payload = replace_vars_in_str(PayloadToken, MapMsg), + Payload = process_payload(PayloadToken, MapMsg), QoS = replace_simple_var(QoSToken, MapMsg), Retain = replace_simple_var(RetainToken, MapMsg), #mqtt_msg{qos = QoS, @@ -82,13 +82,18 @@ to_broker_msg(#{dup := Dup, properties := Props} = MapMsg, #{local_topic := TopicToken, payload := PayloadToken, local_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) -> Topic = replace_vars_in_str(TopicToken, MapMsg), - Payload = replace_vars_in_str(PayloadToken, MapMsg), + Payload = process_payload(PayloadToken, MapMsg), QoS = replace_simple_var(QoSToken, MapMsg), Retain = replace_simple_var(RetainToken, MapMsg), set_headers(Props, emqx_message:set_flags(#{dup => Dup, retain => Retain}, emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload))). +process_payload([], Msg) -> + emqx_json:encode(Msg); +process_payload(Tks, Msg) -> + replace_vars_in_str(Tks, Msg). + %% Replace a string contains vars to another string in which the placeholders are replace by the %% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be: %% "a: 1". From e44f7de59643a4894ed79c8131ce4d72c3799087 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 28 Dec 2021 15:39:27 +0800 Subject: [PATCH 26/99] chore(conf): update hocon to 0.22.1 --- apps/emqx/rebar.config | 2 +- apps/emqx/src/emqx_config.erl | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 330291def..4acb5cc30 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -17,7 +17,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.1"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.16.0"}}} diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index b2154b429..e0e6968f3 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -281,7 +281,7 @@ init_load(SchemaMod, RawConf) when is_map(RawConf) -> maps:with(RootNames, RawConfWithEnvs)). include_dirs() -> - [filename:join(emqx:data_dir(), "configs") ++ "/"]. + [filename:join(emqx:data_dir(), "configs")]. merge_envs(SchemaMod, RawConf) -> Opts = #{logger => fun(_, _) -> ok end, %% everything should have been logged already when check_config diff --git a/rebar.config b/rebar.config index 1894d817d..64a70a467 100644 --- a/rebar.config +++ b/rebar.config @@ -65,7 +65,7 @@ , {system_monitor, {git, "https://github.com/klarna-incubator/system_monitor", {tag, "2.2.0"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.16.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From 0712fc1e75c3048166f56ff53b929113d62bfa0d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 28 Dec 2021 23:50:12 +0800 Subject: [PATCH 27/99] chore(schema): remove emqx_schema:unicode_binary/0 --- apps/emqx/src/emqx_schema.erl | 11 ++--------- apps/emqx_dashboard/src/emqx_dashboard_api.erl | 4 ++-- apps/emqx_management/src/emqx_mgmt_api_app.erl | 2 +- apps/emqx_management/src/emqx_mgmt_api_banned.erl | 6 +++--- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 85b411217..a0bdb5150 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -38,7 +38,6 @@ -type ip_port() :: tuple(). -type cipher() :: map(). -type rfc3339_system_time() :: integer(). --type unicode_binary() :: binary(). -typerefl_from_string({duration/0, emqx_schema, to_duration}). -typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). @@ -52,7 +51,6 @@ -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}). -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}). -typerefl_from_string({rfc3339_system_time/0, emqx_schema, rfc3339_to_system_time}). --typerefl_from_string({unicode_binary/0, emqx_schema, to_unicode_binary}). -export([ validate_heap_size/1 , parse_user_lookup_fun/1 @@ -66,8 +64,7 @@ to_bar_separated_list/1, to_ip_port/1, to_erl_cipher_suite/1, to_comma_separated_atoms/1, - rfc3339_to_system_time/1, - to_unicode_binary/1]). + rfc3339_to_system_time/1]). -behaviour(hocon_schema). @@ -76,8 +73,7 @@ comma_separated_list/0, bar_separated_list/0, ip_port/0, cipher/0, comma_separated_atoms/0, - rfc3339_system_time/0, - unicode_binary/0]). + rfc3339_system_time/0]). -export([namespace/0, roots/0, roots/1, fields/1]). -export([conf_get/2, conf_get/3, keys/2, filter/1]). @@ -1390,9 +1386,6 @@ rfc3339_to_system_time(DateTime) -> {error, bad_rfc3339_timestamp} end. -to_unicode_binary(Str) -> - {ok, unicode:characters_to_binary(Str)}. - to_bar_separated_list(Str) -> {ok, string:tokens(Str, "| ")}. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index 45a3b7c56..ae2eb4e42 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -123,7 +123,7 @@ schema("/users/:username") -> #{in => path, example => <<"admin">>})}], 'requestBody' => [ { description - , mk(emqx_schema:unicode_binary(), + , mk(binary(), #{desc => <<"User description">>, example => <<"administrator">>})} ], responses => #{ @@ -176,7 +176,7 @@ schema("/users/:username/change_pwd") -> fields(user) -> [ {description, - mk(emqx_schema:unicode_binary(), + mk(binary(), #{desc => <<"User description">>, example => "administrator"})}, {username, mk(binary(), diff --git a/apps/emqx_management/src/emqx_mgmt_api_app.erl b/apps/emqx_management/src/emqx_mgmt_api_app.erl index b77f1a214..53ad01485 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_app.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_app.erl @@ -100,7 +100,7 @@ fields(app) -> #{desc => "ApiKey create datetime", example => <<"2021-12-01T00:00:00.000Z">> })}, - {desc, hoconsc:mk(emqx_schema:unicode_binary(), + {desc, hoconsc:mk(binary(), #{example => <<"Note">>, nullable => true})}, {enable, hoconsc:mk(boolean(), #{desc => "Enable/Disable", nullable => true})} ]; diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 911b298a1..c6ff56ad0 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -101,15 +101,15 @@ fields(ban) -> desc => <<"Banned type clientid, username, peerhost">>, nullable => false, example => username})}, - {who, hoconsc:mk(emqx_schema:unicode_binary(), #{ + {who, hoconsc:mk(binary(), #{ desc => <<"Client info as banned type">>, nullable => false, example => <<"Badass坏"/utf8>>})}, - {by, hoconsc:mk(emqx_schema:unicode_binary(), #{ + {by, hoconsc:mk(binary(), #{ desc => <<"Commander">>, nullable => true, example => <<"mgmt_api">>})}, - {reason, hoconsc:mk(emqx_schema:unicode_binary(), #{ + {reason, hoconsc:mk(binary(), #{ desc => <<"Banned reason">>, nullable => true, example => <<"Too many requests">>})}, From b78cfa0a1c81191c9675bea89da05338195294e5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 29 Dec 2021 00:01:36 +0800 Subject: [PATCH 28/99] fix(api_key): support expired_at never expired when undefined --- .../emqx_management/src/emqx_mgmt_api_app.erl | 22 ++++++++++++++----- apps/emqx_management/src/emqx_mgmt_auth.erl | 2 +- .../test/emqx_mgmt_auth_api_SUITE.erl | 19 +++++++++++++++- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_app.erl b/apps/emqx_management/src/emqx_mgmt_api_app.erl index 53ad01485..dfce3cf30 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_app.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_app.erl @@ -91,10 +91,11 @@ fields(app) -> """They are useful for accessing public data anonymously,""" """and are used to associate API requests.""", example => <<"MzAyMjk3ODMwMDk0NjIzOTUxNjcwNzQ0NzQ3MTE2NDYyMDI">>})}, - {expired_at, hoconsc:mk(emqx_schema:rfc3339_system_time(), + {expired_at, hoconsc:mk(hoconsc:union([undefined, emqx_schema:rfc3339_system_time()]), #{desc => "No longer valid datetime", example => <<"2021-12-05T02:01:34.186Z">>, - nullable => true + nullable => true, + default => undefined })}, {created_at, hoconsc:mk(emqx_schema:rfc3339_system_time(), #{desc => "ApiKey create datetime", @@ -136,9 +137,15 @@ api_key(post, #{body := App}) -> #{ <<"name">> := Name, <<"desc">> := Desc0, - <<"expired_at">> := ExpiredAt, <<"enable">> := Enable } = App, + %% undefined is never expired + ExpiredAt0 = maps:get(<<"expired_at">>, App, <<"undefined">>), + ExpiredAt = + case ExpiredAt0 of + <<"undefined">> -> undefined; + _ -> ExpiredAt0 + end, Desc = unicode:characters_to_binary(Desc0, unicode), case emqx_mgmt_auth:create(Name, Enable, ExpiredAt, Desc) of {ok, NewApp} -> {200, format(NewApp)}; @@ -164,8 +171,13 @@ api_key_by_name(put, #{bindings := #{name := Name}, body := Body}) -> {error, not_found} -> {404, <<"NOT_FOUND">>} end. -format(App = #{expired_at := ExpiredAt, created_at := CreateAt}) -> +format(App = #{expired_at := ExpiredAt0, created_at := CreateAt}) -> + ExpiredAt = + case ExpiredAt0 of + undefined -> <<"undefined">>; + _ -> list_to_binary(calendar:system_time_to_rfc3339(ExpiredAt0)) + end, App#{ - expired_at => list_to_binary(calendar:system_time_to_rfc3339(ExpiredAt)), + expired_at => ExpiredAt, created_at => list_to_binary(calendar:system_time_to_rfc3339(CreateAt)) }. diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index ae6b0820d..512ec6b0f 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -37,7 +37,7 @@ api_secret_hash = <<>> :: binary() | '_', enable = true :: boolean() | '_', desc = <<>> :: binary() | '_', - expired_at = 0 :: integer() | '_', + expired_at = 0 :: integer() | undefined | '_', created_at = 0 :: integer() | '_' }). diff --git a/apps/emqx_management/test/emqx_mgmt_auth_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_auth_api_SUITE.erl index 185ad5343..73d4ad566 100644 --- a/apps/emqx_management/test/emqx_mgmt_auth_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_auth_api_SUITE.erl @@ -23,7 +23,7 @@ all() -> [{group, parallel}, {group, sequence}]. suite() -> [{timetrap, {minutes, 1}}]. groups() -> [ - {parallel, [parallel], [t_create, t_update, t_delete, t_authorize]}, + {parallel, [parallel], [t_create, t_update, t_delete, t_authorize, t_create_unexpired_app]}, {sequence, [], [t_create_failed]} ]. @@ -137,7 +137,15 @@ t_authorize(_Config) -> }, ?assertMatch({ok, #{<<"api_key">> := _, <<"enable">> := true}}, update_app(Name, Expired)), ?assertEqual(Unauthorized, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader)), + ok. +t_create_unexpired_app(_Config) -> + Name1 = <<"EMQX-UNEXPIRED-API-KEY-1">>, + Name2 = <<"EMQX-UNEXPIRED-API-KEY-2">>, + {ok, Create1} = create_unexpired_app(Name1, #{}), + ?assertMatch(#{<<"expired_at">> := <<"undefined">>}, Create1), + {ok, Create2} = create_unexpired_app(Name2, #{expired_at => <<"undefined">>}), + ?assertMatch(#{<<"expired_at">> := <<"undefined">>}, Create2), ok. @@ -170,6 +178,15 @@ create_app(Name) -> Error -> Error end. +create_unexpired_app(Name, Params) -> + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Path = emqx_mgmt_api_test_util:api_path(["api_key"]), + App = maps:merge(#{name => Name, desc => <<"Note"/utf8>>, enable => true}, Params), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, App) of + {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + Error -> Error + end. + delete_app(Name) -> DeletePath = emqx_mgmt_api_test_util:api_path(["api_key", Name]), emqx_mgmt_api_test_util:request_api(delete, DeletePath). From 4b6bba11eb3ddec8fbd64745f44b0bc9f42035d5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 28 Dec 2021 23:46:22 +0800 Subject: [PATCH 29/99] feat(trace): struct log for trace --- apps/emqx/include/logger.hrl | 2 +- apps/emqx/src/emqx_broker.erl | 2 +- apps/emqx/src/emqx_channel.erl | 2 +- apps/emqx/src/emqx_connection.erl | 10 +-- apps/emqx/src/emqx_packet.erl | 85 +++++++++++-------- apps/emqx/src/emqx_schema.erl | 17 ++++ apps/emqx/src/emqx_trace/emqx_trace.erl | 15 ++-- .../src/emqx_trace/emqx_trace_formatter.erl | 43 +++++----- .../src/emqx_trace/emqx_trace_handler.erl | 11 ++- apps/emqx/src/emqx_ws_connection.erl | 10 +-- .../emqx_conf/test/emqx_cluster_rpc_SUITE.erl | 14 ++- .../src/emqx_connector_http.erl | 4 +- .../src/emqx_connector_ldap.erl | 9 +- .../src/emqx_connector_mongo.erl | 6 +- .../src/emqx_connector_mqtt.erl | 9 +- .../src/emqx_connector_mysql.erl | 5 +- .../src/emqx_connector_pgsql.erl | 6 +- .../src/emqx_connector_redis.erl | 6 +- .../src/emqx_rule_outputs.erl | 4 +- .../src/emqx_rule_runtime.erl | 2 +- .../src/emqx_rule_sqltester.erl | 2 +- 21 files changed, 154 insertions(+), 110 deletions(-) diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index ecedfafe7..42d598ef9 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -69,7 +69,7 @@ ok end). --define(TRACE(Action, Meta, Msg), emqx_trace:log(Action, Meta, Msg)). +-define(TRACE(Event, Msg, Meta), emqx_trace:log(Event, Msg, Meta)). %% print to 'user' group leader -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)). diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index 6dce69136..dec753fc2 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -205,7 +205,7 @@ publish(Msg) when is_record(Msg, message) -> emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'), case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of #message{headers = #{allow_publish := false}} -> - ?TRACE("NotAllow", #{payload => emqx_message:to_log_map(Msg)}, "message_not_published"), + ?TRACE("MQTT", "msg_publish_not_allowed", #{message => emqx_message:to_log_map(Msg)}), []; Msg1 = #message{topic = Topic} -> emqx_persistent_session:persist_message(Msg1), diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 7fe366ad1..0f83b04ff 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -292,7 +292,7 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) -> fun check_banned/2 ], ConnPkt, Channel#channel{conn_state = connecting}) of {ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} -> - ?TRACE("RECV", #{}, Packet), + ?TRACE("MQTT", "mqtt_packet_received", #{packet => Packet}), NChannel1 = NChannel#channel{ will_msg = emqx_packet:will_msg(NConnPkt), alias_maximum = init_alias_maximum(NConnPkt, ClientInfo) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index b8a6b4b7b..37e15f522 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -526,7 +526,7 @@ handle_msg({connack, ConnAck}, State) -> handle_outgoing(ConnAck, State); handle_msg({close, Reason}, State) -> - ?TRACE("CLOSE", #{reason => Reason}, "force_socket_close"), + ?TRACE("SOCKET", "socket_force_closed", #{reason => Reason}), handle_info({sock_closed, Reason}, close_socket(State)); handle_msg({event, connected}, State = #state{channel = Channel}) -> @@ -565,7 +565,7 @@ terminate(Reason, State = #state{channel = Channel, transport = Transport, emqx_congestion:cancel_alarms(Socket, Transport, Channel1), emqx_channel:terminate(Reason, Channel1), close_socket_ok(State), - ?TRACE("TERMINATE", #{reason => Reason}, "terminated") + ?TRACE("SOCKET", "tcp_socket_terminated", #{reason => Reason}) catch E : C : S -> ?tp(warning, unclean_terminate, #{exception => E, context => C, stacktrace => S}) @@ -715,7 +715,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) -> handle_incoming(Packet, State) when is_record(Packet, mqtt_packet) -> ok = inc_incoming_stats(Packet), - ?TRACE("RECV", #{}, Packet), + ?TRACE("MQTT", "mqtt_packet_received", #{packet => Packet}), with_channel(handle_in, [Packet], State); handle_incoming(FrameError, State) -> @@ -754,13 +754,13 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) -> <<>> -> ?SLOG(warning, #{ msg => "packet_is_discarded", reason => "frame_is_too_large", - packet => emqx_packet:format(Packet) + packet => emqx_packet:format(Packet, null) }), ok = emqx_metrics:inc('delivery.dropped.too_large'), ok = emqx_metrics:inc('delivery.dropped'), <<>>; Data -> - ?TRACE("SEND", #{}, Packet), + ?TRACE("MQTT", "mqtt_packet_sent", #{packet => Packet}), ok = inc_outgoing_stats(Packet), Data catch diff --git a/apps/emqx/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl index 02165a6b5..53e4eabfe 100644 --- a/apps/emqx/src/emqx_packet.erl +++ b/apps/emqx/src/emqx_packet.erl @@ -44,7 +44,9 @@ , will_msg/1 ]). --export([format/1]). +-export([ format/1 + , format/2 + ]). -define(TYPE_NAMES, { 'CONNECT' @@ -435,25 +437,28 @@ will_msg(#mqtt_packet_connect{clientid = ClientId, %% @doc Format packet -spec(format(emqx_types:packet()) -> iolist()). -format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}) -> - format_header(Header, format_variable(Variable, Payload)). +format(Packet) -> format(Packet, emqx_trace_handler:payload_encode()). + +%% @doc Format packet +-spec(format(emqx_types:packet(), hex | text | null) -> iolist()). +format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}, PayloadEncode) -> + HeaderIO = format_header(Header), + case format_variable(Variable, Payload, PayloadEncode) of + "" -> HeaderIO; + VarIO -> [HeaderIO,",", VarIO] + end. format_header(#mqtt_packet_header{type = Type, dup = Dup, qos = QoS, - retain = Retain}, S) -> - S1 = case S == undefined of - true -> <<>>; - false -> [", ", S] - end, - io_lib:format("~ts(Q~p, R~p, D~p~ts)", [type_name(Type), QoS, i(Retain), i(Dup), S1]). + retain = Retain}) -> + io_lib:format("~ts(Q~p, R~p, D~p)", [type_name(Type), QoS, i(Retain), i(Dup)]). -format_variable(undefined, _) -> - undefined; -format_variable(Variable, undefined) -> - format_variable(Variable); -format_variable(Variable, Payload) -> - io_lib:format("~ts, Payload=~ts", [format_variable(Variable), Payload]). +format_variable(undefined, _, _) -> ""; +format_variable(Variable, undefined, PayloadEncode) -> + format_variable(Variable, PayloadEncode); +format_variable(Variable, Payload, PayloadEncode) -> + [format_variable(Variable, PayloadEncode), format_payload(Payload, PayloadEncode)]. format_variable(#mqtt_packet_connect{ proto_ver = ProtoVer, @@ -467,55 +472,63 @@ format_variable(#mqtt_packet_connect{ will_topic = WillTopic, will_payload = WillPayload, username = Username, - password = Password}) -> - Format = "ClientId=~ts, ProtoName=~ts, ProtoVsn=~p, CleanStart=~ts, KeepAlive=~p, Username=~ts, Password=~ts", - Args = [ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)], - {Format1, Args1} = if - WillFlag -> {Format ++ ", Will(Q~p, R~p, Topic=~ts, Payload=~0p)", - Args ++ [WillQoS, i(WillRetain), WillTopic, WillPayload]}; - true -> {Format, Args} - end, - io_lib:format(Format1, Args1); + password = Password}, + PayloadEncode) -> + Base = io_lib:format( + "ClientId=~ts, ProtoName=~ts, ProtoVsn=~p, CleanStart=~ts, KeepAlive=~p, Username=~ts, Password=~ts", + [ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)]), + case WillFlag of + true -> + [Base, io_lib:format(", Will(Q~p, R~p, Topic=~ts ", + [WillQoS, i(WillRetain), WillTopic]), + format_payload(WillPayload, PayloadEncode), ")"]; + false -> + Base + end; format_variable(#mqtt_packet_disconnect - {reason_code = ReasonCode}) -> + {reason_code = ReasonCode}, _) -> io_lib:format("ReasonCode=~p", [ReasonCode]); format_variable(#mqtt_packet_connack{ack_flags = AckFlags, - reason_code = ReasonCode}) -> + reason_code = ReasonCode}, _) -> io_lib:format("AckFlags=~p, ReasonCode=~p", [AckFlags, ReasonCode]); format_variable(#mqtt_packet_publish{topic_name = TopicName, - packet_id = PacketId}) -> + packet_id = PacketId}, _) -> io_lib:format("Topic=~ts, PacketId=~p", [TopicName, PacketId]); format_variable(#mqtt_packet_puback{packet_id = PacketId, - reason_code = ReasonCode}) -> + reason_code = ReasonCode}, _) -> io_lib:format("PacketId=~p, ReasonCode=~p", [PacketId, ReasonCode]); format_variable(#mqtt_packet_subscribe{packet_id = PacketId, - topic_filters = TopicFilters}) -> + topic_filters = TopicFilters}, _) -> io_lib:format("PacketId=~p, TopicFilters=~0p", [PacketId, TopicFilters]); format_variable(#mqtt_packet_unsubscribe{packet_id = PacketId, - topic_filters = Topics}) -> + topic_filters = Topics}, _) -> io_lib:format("PacketId=~p, TopicFilters=~0p", [PacketId, Topics]); format_variable(#mqtt_packet_suback{packet_id = PacketId, - reason_codes = ReasonCodes}) -> + reason_codes = ReasonCodes}, _) -> io_lib:format("PacketId=~p, ReasonCodes=~p", [PacketId, ReasonCodes]); -format_variable(#mqtt_packet_unsuback{packet_id = PacketId}) -> +format_variable(#mqtt_packet_unsuback{packet_id = PacketId}, _) -> io_lib:format("PacketId=~p", [PacketId]); -format_variable(#mqtt_packet_auth{reason_code = ReasonCode}) -> +format_variable(#mqtt_packet_auth{reason_code = ReasonCode}, _) -> io_lib:format("ReasonCode=~p", [ReasonCode]); -format_variable(PacketId) when is_integer(PacketId) -> +format_variable(PacketId, _) when is_integer(PacketId) -> io_lib:format("PacketId=~p", [PacketId]). -format_password(undefined) -> undefined; -format_password(_Password) -> '******'. +format_password(undefined) -> "undefined"; +format_password(_Password) -> "******". + +format_payload(Payload, text) -> ["Payload=", io_lib:format("~ts", [Payload])]; +format_payload(Payload, hex) -> ["Payload(hex)=", binary:encode_hex(Payload)]; +format_payload(_, null) -> "Payload=******". i(true) -> 1; i(false) -> 0; diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 85b411217..cbf21683d 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -184,6 +184,12 @@ roots(low) -> , {"latency_stats", sc(ref("latency_stats"), #{})} + , {"trace", + sc(ref("trace"), + #{desc => """ +Real-time filtering logs for the ClientID or Topic or IP for debugging. +""" + })} ]. fields("persistent_session_store") -> @@ -981,6 +987,17 @@ when deactivated, but after the retention time. fields("latency_stats") -> [ {"samples", sc(integer(), #{default => 10, desc => "the number of smaples for calculate the average latency of delivery"})} + ]; +fields("trace") -> + [ {"payload_encode", sc(hoconsc:enum([hex, text, null]), #{ + default => text, + desc => """ +Determine the format of the payload format in the trace file.
+- `text`: Text-based protocol or plain text protocol. It is recommended when payload is json encode.
+- `hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.
+- `null`: Don't show payload in trace log file. + """ + })} ]. mqtt_listener() -> diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index f4679e073..7bca3ca3c 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -84,22 +84,22 @@ mnesia(boot) -> publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore; publish(#message{from = From, topic = Topic, payload = Payload}) when is_binary(From); is_atom(From) -> - ?TRACE("PUBLISH", #{topic => Topic}, {publish, Payload}). + ?TRACE("PUBLISH", "publish_to", #{topic => Topic, payload => Payload}). subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore; subscribe(Topic, SubId, SubOpts) -> - ?TRACE("SUBSCRIBE", #{topic => Topic}, {subscribe, SubId, SubOpts}). + ?TRACE("SUBSCRIBE", "subscribe", #{topic => Topic, sub_opts => SubOpts, sub_id => SubId}). unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore; unsubscribe(Topic, SubOpts) -> - ?TRACE("UNSUBSCRIBE", #{topic => Topic}, {unsubscribe, SubOpts}). + ?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}). -log(Action, Meta0, Msg) -> +log(Event, Msg, Meta0) -> case persistent_term:get(?TRACE_FILTER, undefined) of undefined -> ok; List -> Meta = maps:merge(logger:get_process_metadata(), Meta0), - Log = #{level => trace, action => Action, meta => Meta, msg => Msg}, + Log = #{level => trace, event => Event, meta => Meta, msg => Msg}, log_filter(List, Log) end. @@ -112,11 +112,12 @@ log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) -> case logger_config:get(ets:whereis(logger), Id) of {ok, #{module := Module} = HandlerConfig0} -> HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0), + io:format("~p~n", [{Module, Log, HandlerConfig}]), try Module:log(Log, HandlerConfig) catch C:R:S -> case logger:remove_handler(Id) of ok -> - logger:internal_log(error, {removed_failing_handler, Id}); + logger:internal_log(error, {removed_failing_handler, Id, C, R, S}); {error,{not_found,_}} -> %% Probably already removed by other client %% Don't report again @@ -528,5 +529,5 @@ update_trace_handler() -> filter_cli_handler(Names) -> lists:filter(fun(Name) -> - notmatch =:= re:run(Name, "^CLI-+.", []) + nomatch =:= re:run(Name, "^CLI-+.", []) end, Names). diff --git a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl index 1b37b7130..a9c4ca31d 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl @@ -22,35 +22,36 @@ -spec format(LogEvent, Config) -> unicode:chardata() when LogEvent :: logger:log_event(), Config :: logger:config(). -format(#{level := trace, msg := Msg, meta := Meta, action := Action}, _Config) -> +format(#{level := trace, event := Event, meta := Meta, msg := Msg}, + #{payload_encode := PEncode}) -> Time = calendar:system_time_to_rfc3339(erlang:system_time(second)), ClientId = maps:get(clientid, Meta, ""), Peername = maps:get(peername, Meta, ""), - MsgBin = format_msg(Msg), - MetaBin = format_map(maps:without([clientid, peername], Meta)), - [Time, " [", Action, "] ", ClientId, "@", Peername, " ", MsgBin, " ( ", - MetaBin, ")\n"]; + MetaBin = format_meta(Meta, PEncode), + [Time, " [", Event, "] ", ClientId, "@", Peername, " msg: ", Msg, MetaBin, "\n"]; format(Event, Config) -> emqx_logger_textfmt:format(Event, Config). -format_msg(Bin)when is_binary(Bin) -> Bin; -format_msg(List) when is_list(List) -> List; -format_msg({publish, Payload}) -> - io_lib:format("Publish Payload:(~ts) TO ", [Payload]); -format_msg({subscribe, SubId, SubOpts}) -> - [io_lib:format("SUBSCRIBE ~ts, Opts( ", [SubId]), - format_map(SubOpts), ")"]; -format_msg({unsubscribe, SubOpts}) -> - [io_lib:format("UNSUBSCRIBE ~ts, Opts( ", [maps:get(subid, SubOpts, "undefined")]), - format_map(maps:without([subid], SubOpts)), ")"]; -format_msg(Packet) -> - emqx_packet:format(Packet). +format_meta(Meta0, Encode) -> + Packet = format_packet(maps:get(packet, Meta0, undefined), Encode), + Payload = format_payload(maps:get(payload, Meta0, undefined), Encode), + Meta1 = maps:without([msg, clientid, peername, packet, payload], Meta0), + case Meta1 =:= #{} of + true -> [Packet, Payload]; + false -> + Meta2 = lists:map(fun({K, V}) -> [to_iolist(K), ": ", to_iolist(V)] end, + maps:to_list(Meta1)), + [Packet, ", ", lists:join(",", Meta2), Payload] + end. -format_map(Map) -> - maps:fold(fun(K, V, Acc) -> - [to_iolist(K), ":", to_iolist(V), " "|Acc] - end, [], Map). +format_packet(undefined, _) -> ""; +format_packet(Packet, Encode) -> [", packet: ", emqx_packet:format(Packet, Encode)]. + +format_payload(undefined, _) -> ""; +format_payload(Payload, text) -> [", payload: ", io_lib:format("~ts", [Payload])]; +format_payload(Payload, hex) -> [", payload(hex): ", binary:encode_hex(Payload)]; +format_payload(_, null) -> ", payload=******". to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom); to_iolist(Int) when is_integer(Int) -> integer_to_list(Int); diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl index 9c991301c..d1fbd20b8 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl @@ -35,8 +35,10 @@ , filter_topic/2 , filter_ip_address/2 ]). +-export([template/1]). -export([handler_id/2]). +-export([payload_encode/0]). -type tracer() :: #{ name := binary(), @@ -145,13 +147,14 @@ filters(#{type := topic, filter := Filter, name := Name}) -> filters(#{type := ip_address, filter := Filter, name := Name}) -> [{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}]. -formatter(#{type := Type}) -> +formatter(#{type := _Type}) -> {emqx_trace_formatter, #{ - template => template(Type), + template => [], single_line => false, max_size => unlimited, - depth => unlimited + depth => unlimited, + payload_encode => payload_encode() } }. @@ -181,6 +184,8 @@ filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) Acc end. +payload_encode() -> emqx_config:get([trace, payload_encode], text). + handler_id(Name, Type) -> try do_handler_id(Name, Type) diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index 6e6bc1c90..f70d7ac6c 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -431,11 +431,11 @@ websocket_info(Info, State) -> websocket_close({_, ReasonCode, _Payload}, State) when is_integer(ReasonCode) -> websocket_close(ReasonCode, State); websocket_close(Reason, State) -> - ?TRACE("CLOSED", #{transport => websocket, reason => Reason}, "websocket_closed"), + ?TRACE("SOCKET", "websocket_closed", #{reason => Reason}), handle_info({sock_closed, Reason}, State). terminate(Reason, _Req, #state{channel = Channel}) -> - ?TRACE("TERMINATE", #{transport => websocket, reason => Reason}, "webscoket_terminated"), + ?TRACE("SOCKET", "websocket_terminated", #{reason => Reason}), emqx_channel:terminate(Reason, Channel); terminate(_Reason, _Req, _UnExpectedState) -> @@ -479,7 +479,7 @@ handle_info({connack, ConnAck}, State) -> return(enqueue(ConnAck, State)); handle_info({close, Reason}, State) -> - ?TRACE("CLOSE", #{reason => Reason}, "force_socket_close"), + ?TRACE("SOCKET", "socket_force_closed", #{reason => Reason}), return(enqueue({close, Reason}, State)); handle_info({event, connected}, State = #state{channel = Channel}) -> @@ -662,7 +662,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) -> handle_incoming(Packet, State = #state{listener = {Type, Listener}}) when is_record(Packet, mqtt_packet) -> - ?TRACE("RECV", #{transport => websocket}, Packet), + ?TRACE("WS-MQTT", "mqtt_packet_received", #{packet => Packet}), ok = inc_incoming_stats(Packet), NState = case emqx_pd:get_counter(incoming_pubs) > get_active_n(Type, Listener) of @@ -726,7 +726,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) -> ok = emqx_metrics:inc('delivery.dropped.too_large'), ok = emqx_metrics:inc('delivery.dropped'), <<>>; - Data -> ?TRACE("SEND", #{transport => websocket}, Packet), + Data -> ?TRACE("WS-MQTT", "mqtt_packet_sent", #{packet => Packet}), ok = inc_outgoing_stats(Packet), Data catch diff --git a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl index ad74faf99..66b05c95a 100644 --- a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl +++ b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl @@ -74,9 +74,19 @@ t_base_test(_Config) -> ?assertEqual(node(), maps:get(initiator, Query)), ?assert(maps:is_key(created_at, Query)), ?assertEqual(ok, receive_msg(3, test)), + ?assertEqual({ok, 2, ok}, emqx_cluster_rpc:multicall(M, F, A)), {atomic, Status} = emqx_cluster_rpc:status(), - ?assertEqual(3, length(Status)), - ?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 1 end, Status)), + case length(Status) =:= 3 of + true -> ?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 2 end, Status)); + false -> + %% wait for mnesia to write in. + ct:sleep(42), + {atomic, Status1} = emqx_cluster_rpc:status(), + ct:pal("status: ~p", Status), + ct:pal("status1: ~p", Status1), + ?assertEqual(3, length(Status1)), + ?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 2 end, Status)) + end, ok. t_commit_fail_test(_Config) -> diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index b54d87f12..7d7771503 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -199,8 +199,8 @@ on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) -> on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State); on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery, #{pool_name := PoolName, base_path := BasePath} = State) -> - ?TRACE("QUERY", #{request => Request, connector => InstId, state => State}, - "http connector received request"), + ?TRACE("QUERY", "http_connector_received", + #{request => Request, connector => InstId, state => State}), NRequest = update_path(BasePath, Request), case Result = ehttpc:request(case KeyOrNum of undefined -> PoolName; diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index 8aa1f9319..97e963f18 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -87,16 +87,15 @@ on_stop(InstId, #{poolname := PoolName}) -> on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) -> Request = {Base, Filter, Attributes}, - ?TRACE("QUERY", #{request => Request, connector => InstId, state => State}, - "ldap connector received request"), + ?TRACE("QUERY", "ldap_connector_received", + #{request => Request, connector => InstId, state => State}), case Result = ecpool:pick_and_do( PoolName, {?MODULE, search, [Base, Filter, Attributes]}, no_handover) of {error, Reason} -> - ?SLOG(error, #{msg => "ldap connector do request failed", - request => Request, connector => InstId, - reason => Reason}), + ?SLOG(error, #{msg => "ldap_connector_do_request_failed", + request => Request, connector => InstId, reason => Reason}), emqx_resource:query_failed(AfterQuery); _ -> emqx_resource:query_success(AfterQuery) diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index d2594ab93..eacb3ec2d 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -137,13 +137,13 @@ on_query(InstId, AfterQuery, #{poolname := PoolName} = State) -> Request = {Action, Collection, Selector, Docs}, - ?TRACE("QUERY", #{request => Request, connector => InstId, state => State}, - "mongodb connector received request"), + ?TRACE("QUERY", "mongodb_connector_received", + #{request => Request, connector => InstId, state => State}), case ecpool:pick_and_do(PoolName, {?MODULE, mongo_query, [Action, Collection, Selector, Docs]}, no_handover) of {error, Reason} -> - ?SLOG(error, #{msg => "mongodb connector do query failed", + ?SLOG(error, #{msg => "mongodb_connector_do_query_failed", request => Request, reason => Reason, connector => InstId}), emqx_resource:query_failed(AfterQuery), diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 6d620cc14..27d001806 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -118,7 +118,7 @@ on_message_received(Msg, HookPoint) -> %% =================================================================== on_start(InstId, Conf) -> InstanceId = binary_to_atom(InstId, utf8), - ?SLOG(info, #{msg => "starting mqtt connector", + ?SLOG(info, #{msg => "starting_mqtt_connector", connector => InstanceId, config => Conf}), BasicConf = basic_config(Conf), BridgeConf = BasicConf#{ @@ -139,19 +139,18 @@ on_start(InstId, Conf) -> end. on_stop(_InstId, #{name := InstanceId}) -> - ?SLOG(info, #{msg => "stopping mqtt connector", + ?SLOG(info, #{msg => "stopping_mqtt_connector", connector => InstanceId}), case ?MODULE:drop_bridge(InstanceId) of ok -> ok; {error, not_found} -> ok; {error, Reason} -> - ?SLOG(error, #{msg => "stop mqtt connector", + ?SLOG(error, #{msg => "stop_mqtt_connector", connector => InstanceId, reason => Reason}) end. on_query(_InstId, {send_message, Msg}, AfterQuery, #{name := InstanceId}) -> - ?TRACE("QUERY", #{message => Msg, connector => InstanceId}, - "send msg to remote node"), + ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}), emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg), emqx_resource:query_success(AfterQuery). diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index def3904b4..ae8239936 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -85,14 +85,13 @@ on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) -> on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := _PoolName} = State) -> on_query(InstId, {sql, SQL, Params, default_timeout}, AfterQuery, State); on_query(InstId, {sql, SQL, Params, Timeout}, AfterQuery, #{poolname := PoolName} = State) -> - ?TRACE("QUERY", #{connector => InstId, sql => SQL, state => State}, - "mysql connector received sql query"), + ?TRACE("QUERY", "mysql_connector_received", #{connector => InstId, sql => SQL, state => State}), case Result = ecpool:pick_and_do( PoolName, {mysql, query, [SQL, Params, Timeout]}, no_handover) of {error, Reason} -> - ?SLOG(error, #{msg => "mysql connector do sql query failed", + ?SLOG(error, #{msg => "mysql_connector_do_sql_query_failed", connector => InstId, sql => SQL, reason => Reason}), emqx_resource:query_failed(AfterQuery); _ -> diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index ac864a45d..9b6f559b4 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -83,12 +83,12 @@ on_stop(InstId, #{poolname := PoolName}) -> on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) -> on_query(InstId, {sql, SQL, []}, AfterQuery, State); on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) -> - ?TRACE("QUERY", #{connector => InstId, sql => SQL, state => State}, - "postgresql connector received sql query"), + ?TRACE("QUERY", "postgresql_connector_received", + #{connector => InstId, sql => SQL, state => State}), case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [SQL, Params]}, no_handover) of {error, Reason} -> ?SLOG(error, #{ - msg => "postgresql connector do sql query failed", + msg => "postgresql_connector_do_sql_query_failed", connector => InstId, sql => SQL, reason => Reason}), emqx_resource:query_failed(AfterQuery); _ -> diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 61b716b8b..94f4eca3e 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -125,15 +125,15 @@ on_stop(InstId, #{poolname := PoolName}) -> emqx_plugin_libs_pool:stop_pool(PoolName). on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := Type} = State) -> - ?TRACE("QUERY", #{connector => InstId, sql => Command, state => State}, - "redis connector received cmd query"), + ?TRACE("QUERY", "redis_connector_received", + #{connector => InstId, sql => Command, state => State}), Result = case Type of cluster -> eredis_cluster:q(PoolName, Command); _ -> ecpool:pick_and_do(PoolName, {?MODULE, cmd, [Type, Command]}, no_handover) end, case Result of {error, Reason} -> - ?SLOG(error, #{msg => "redis connector do cmd query failed", + ?SLOG(error, #{msg => "redis_connector_do_cmd_query_failed", connector => InstId, sql => Command, reason => Reason}), emqx_resource:query_failed(AfterCommand); _ -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_outputs.erl b/apps/emqx_rule_engine/src/emqx_rule_outputs.erl index 70aa68cf5..d02f62d70 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_outputs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_outputs.erl @@ -85,7 +85,7 @@ republish(Selected, #{flags := Flags, metadata := #{rule_id := RuleId}}, Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected), QoS = replace_simple_var(QoSTks, Selected, 0), Retain = replace_simple_var(RetainTks, Selected, false), - ?TRACE("REPUBLISH", #{topic => Topic, payload => Payload}, "republish message"), + ?TRACE("RULE", "republish_message", #{topic => Topic, payload => Payload}), safe_publish(RuleId, Topic, QoS, Flags#{retain => Retain}, Payload); %% in case this is a "$events/" event @@ -99,7 +99,7 @@ republish(Selected, #{metadata := #{rule_id := RuleId}}, Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected), QoS = replace_simple_var(QoSTks, Selected, 0), Retain = replace_simple_var(RetainTks, Selected, false), - ?TRACE("REPUBLISH", #{topic => Topic, payload => Payload}, "republish"), + ?TRACE("RULE", "republish_message_with_flags", #{topic => Topic, payload => Payload}), safe_publish(RuleId, Topic, QoS, #{retain => Retain}, Payload). %%-------------------------------------------------------------------- diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index c61296d87..60a7cbaad 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -248,7 +248,7 @@ handle_output(OutId, Selected, Envs) -> end. do_handle_output(BridgeId, Selected, _Envs) when is_binary(BridgeId) -> - ?TRACE("SEND", #{bridge_id => BridgeId}, "output to bridge"), + ?TRACE("BRIDGE", "output_to_bridge", #{bridge_id => BridgeId}), emqx_bridge:send_message(BridgeId, Selected); do_handle_output(#{mod := Mod, func := Func, args := Args}, Selected, Envs) -> Mod:Func(Selected, Envs, Args). diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index 7a9da25a2..cd4d0ce6b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -77,7 +77,7 @@ flatten([D1 | L]) when is_list(D1) -> D1 ++ flatten(L). echo_action(Data, Envs) -> - ?TRACE("TEST", #{data => Data, envs => Envs}, "testing_rule_sql_ok"), + ?TRACE("RULE", "testing_rule_sql_ok", #{data => Data, envs => Envs}), Data. fill_default_values(Event, Context) -> From 8b5b3a448a4c894e2a7d245bcab45c85d24191d4 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 29 Dec 2021 01:15:47 +0800 Subject: [PATCH 30/99] fix(test): trace_handler ct fail --- apps/emqx/etc/emqx.conf.rendered | 1671 +++++++++++++++++ apps/emqx/src/emqx_trace/emqx_trace.erl | 1 - apps/emqx/test/emqx_trace_handler_SUITE.erl | 84 +- .../src/emqx_mgmt_api_trace.erl | 4 +- .../etc/emqx_modules.conf.rendered | 50 + 5 files changed, 1768 insertions(+), 42 deletions(-) create mode 100644 apps/emqx/etc/emqx.conf.rendered create mode 100644 apps/emqx_modules/etc/emqx_modules.conf.rendered diff --git a/apps/emqx/etc/emqx.conf.rendered b/apps/emqx/etc/emqx.conf.rendered new file mode 100644 index 000000000..afa640621 --- /dev/null +++ b/apps/emqx/etc/emqx.conf.rendered @@ -0,0 +1,1671 @@ +##================================================================== +## Listeners +##================================================================== +## MQTT/TCP - TCP Listeners for MQTT Protocol +## syntax: listeners.tcp. +## example: listeners.tcp.my_tcp_listener +listeners.tcp.default { + ## The IP address and port that the listener will bind. + ## + ## @doc listeners.tcp..bind + ## ValueType: IPAddress | Port | IPAddrPort + ## Required: true + ## Examples: 1883, 127.0.0.1:1883, ::1:1883 + bind = "0.0.0.0:1883" + + ## The configuration zone this listener is using. + ## If not set, the global configs are used for this listener. + ## + ## See `zones.` for more details. + ## + ## @doc listeners.tcp..zone + ## ValueType: String + ## Required: false + #zone = default + + ## The size of the acceptor pool for this listener. + ## + ## @doc listeners.tcp..acceptors + ## ValueType: Number + ## Default: 16 + acceptors = 16 + + ## Maximum number of concurrent connections. + ## + ## @doc listeners.tcp..max_connections + ## ValueType: Number | infinity + ## Default: infinity + max_connections = 1024000 + + ## The access control rules for this listener. + ## + ## See: https://github.com/emqtt/esockd#allowdeny + ## + ## @doc listeners.tcp..access_rules + ## ValueType: Array + ## Default: [] + ## Examples: + ## access_rules: [ + ## "deny 192.168.0.0/24", + ## "all all" + ## ] + access_rules = [ + "allow all" + ] + + ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed + ## behind HAProxy or Nginx. + ## + ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ + ## + ## @doc listeners.tcp..proxy_protocol + ## ValueType: Boolean + ## Default: false + proxy_protocol = false + + ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection + ## if no proxy protocol packet received within the timeout. + ## + ## @doc listeners.tcp..proxy_protocol_timeout + ## ValueType: Duration + ## Default: 3s + proxy_protocol_timeout = 3s + + ## When publishing or subscribing, prefix all topics with a mountpoint string. + ## The prefixed string will be removed from the topic name when the message + ## is delivered to the subscriber. The mountpoint is a way that users can use + ## to implement isolation of message routing between different listeners. + ## + ## For example if a clientA subscribes to "t" with `listeners.tcp..mountpoint` + ## set to "some_tenant", then the client accually subscribes to the topic + ## "some_tenant/t". Similarly if another clientB (connected to the same listener + ## with the clientA) send a message to topic "t", the message is accually route + ## to all the clients subscribed "some_tenant/t", so clientA will receive the + ## message, with topic name "t". + ## + ## Set to "" to disable the feature. + ## + ## Variables in mountpoint string: + ## - ${clientid}: clientid + ## - ${username}: username + ## + ## @doc listeners.tcp..mountpoint + ## ValueType: String + ## Default: "" + mountpoint = "" + + ## TCP options + ## See ${example_common_tcp_options} for more information + tcp.backlog = 1024 + tcp.buffer = 4KB +} + +## MQTT/SSL - SSL Listeners for MQTT Protocol +## syntax: listeners.ssl. +## example: listeners.ssl.my_ssl_listener +listeners.ssl.default { + ## The IP address and port that the listener will bind. + ## + ## @doc listeners.ssl..bind + ## ValueType: IPAddress | Port | IPAddrPort + ## Required: true + ## Examples: 8883, 127.0.0.1:8883, ::1:8883 + bind = "0.0.0.0:8883" + + ## The configuration zone this listener is using. + ## If not set, the global configs are used for this listener. + ## + ## See `zones.` for more details. + ## + ## @doc listeners.ssl..zone + ## ValueType: String + ## Required: false + #zone = default + + ## The size of the acceptor pool for this listener. + ## + ## @doc listeners.ssl..acceptors + ## ValueType: Number + ## Default: 16 + acceptors = 16 + + ## Maximum number of concurrent connections. + ## + ## @doc listeners.ssl..max_connections + ## ValueType: Number | infinity + ## Default: infinity + max_connections = 512000 + + ## The access control rules for this listener. + ## + ## See: https://github.com/emqtt/esockd#allowdeny + ## + ## @doc listeners.ssl..access_rules + ## ValueType: Array + ## Default: [] + ## Examples: + ## access_rules: [ + ## "deny 192.168.0.0/24", + ## "all all" + ## ] + access_rules = [ + "allow all" + ] + + ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed + ## behind HAProxy or Nginx. + ## + ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ + ## + ## @doc listeners.ssl..proxy_protocol + ## ValueType: Boolean + ## Default: true + proxy_protocol = false + + ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection + ## if no proxy protocol packet received within the timeout. + ## + ## @doc listeners.ssl..proxy_protocol_timeout + ## ValueType: Duration + ## Default: 3s + proxy_protocol_timeout = 3s + + ## When publishing or subscribing, prefix all topics with a mountpoint string. + ## The prefixed string will be removed from the topic name when the message + ## is delivered to the subscriber. The mountpoint is a way that users can use + ## to implement isolation of message routing between different listeners. + ## + ## For example if a clientA subscribes to "t" with `listeners.ssl..mountpoint` + ## set to "some_tenant", then the client accually subscribes to the topic + ## "some_tenant/t". Similarly if another clientB (connected to the same listener + ## with the clientA) send a message to topic "t", the message is accually route + ## to all the clients subscribed "some_tenant/t", so clientA will receive the + ## message, with topic name "t". + ## + ## Set to "" to disable the feature. + ## + ## Variables in mountpoint string: + ## - ${clientid}: clientid + ## - ${username}: username + ## + ## @doc listeners.ssl..mountpoint + ## ValueType: String + ## Default: "" + mountpoint = "" + + ## SSL options + ssl.keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem" + ssl.certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem" + ssl.cacertfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem" + + # ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + # TLS 1.3: "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256" + # TLS 1-1.2 "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" + # PSK: "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" + # NOTE: If PSK cipher-suites are intended, tlsv1.3 should not be enabled in 'versions' config + # ssl.ciphers = "" + + ## TCP options + ## See ${example_common_tcp_options} for more information + tcp.backlog = 1024 + tcp.buffer = 4KB +} + +## MQTT/QUIC - QUIC Listeners for MQTT Protocol +## syntax: listeners.quic. +## example: listeners.quic.my_quic_listener +listeners.quic.default { + ## The IP address and port that the listener will bind. + ## + ## @doc listeners.quic..bind + ## ValueType: IPAddress | Port | IPAddrPort + ## Required: true + ## Examples: 14567, 127.0.0.1:14567, ::1:14567 + bind = "0.0.0.0:14567" + + ## The configuration zone this listener is using. + ## If not set, the global configs are used for this listener. + ## + ## See `zones.` for more details. + ## NOTE: This is a cluster-wide configuration. + ## It requires all nodes to be stopped before changing it. + ## + ## @doc listeners.quic..zone + ## ValueType: String + ## Required: false + #zone = default + + ## The size of the acceptor pool for this listener. + ## + ## @doc listeners.quic..acceptors + ## ValueType: Number + ## Default: 16 + acceptors = 16 + + ## Maximum number of concurrent connections. + ## + ## @doc listeners.quic..max_connections + ## ValueType: Number | infinity + ## Default: infinity + max_connections = 1024000 + + ## Path to the file containing the user's private PEM-encoded key. + ## + ## @doc listeners.quic..keyfile + ## ValueType: String + ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem" + keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem" + + ## Path to a file containing the user certificate. + ## + ## @doc listeners.quic..certfile + ## ValueType: String + ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem" + certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem" + + ## When publishing or subscribing, prefix all topics with a mountpoint string. + ## The prefixed string will be removed from the topic name when the message + ## is delivered to the subscriber. The mountpoint is a way that users can use + ## to implement isolation of message routing between different listeners. + ## + ## For example if a clientA subscribes to "t" with `listeners.quic..mountpoint` + ## set to "some_tenant", then the client accually subscribes to the topic + ## "some_tenant/t". Similarly if another clientB (connected to the same listener + ## with the clientA) send a message to topic "t", the message is accually route + ## to all the clients subscribed "some_tenant/t", so clientA will receive the + ## message, with topic name "t". + ## + ## Set to "" to disable the feature. + ## + ## Variables in mountpoint string: + ## - ${clientid}: clientid + ## - ${username}: username + ## + ## @doc listeners.quic..mountpoint + ## ValueType: String + ## Default: "" + mountpoint = "" +} + +## MQTT/WS - Websocket Listeners for MQTT Protocol +## syntax: listeners.ws. +## example: listeners.ws.my_ws_listener +listeners.ws.default { + ## The IP address and port that the listener will bind. + ## + ## @doc listeners.ws..bind + ## ValueType: IPAddress | Port | IPAddrPort + ## Required: true + ## Examples: 8083, 127.0.0.1:8083, ::1:8083 + bind = "0.0.0.0:8083" + + ## The configuration zone this listener is using. + ## If not set, the global configs are used for this listener. + ## + ## See `zones.` for more details. + ## + ## @doc listeners.ws..zone + ## ValueType: String + ## Required: false + #zone = default + + ## The size of the acceptor pool for this listener. + ## + ## @doc listeners.ws..acceptors + ## ValueType: Number + ## Default: 16 + acceptors = 16 + + ## Maximum number of concurrent connections. + ## + ## @doc listeners.ws..max_connections + ## ValueType: Number | infinity + ## Default: infinity + max_connections = 1024000 + + ## The access control rules for this listener. + ## + ## See: https://github.com/emqtt/esockd#allowdeny + ## + ## @doc listeners.ws..access_rules + ## ValueType: Array + ## Default: [] + ## Examples: + ## access_rules: [ + ## "deny 192.168.0.0/24", + ## "all all" + ## ] + access_rules = [ + "allow all" + ] + + ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed + ## behind HAProxy or Nginx. + ## + ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ + ## + ## @doc listeners.ws..proxy_protocol + ## ValueType: Boolean + ## Default: true + proxy_protocol = false + + ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection + ## if no proxy protocol packet received within the timeout. + ## + ## @doc listeners.ws..proxy_protocol_timeout + ## ValueType: Duration + ## Default: 3s + proxy_protocol_timeout = 3s + + ## When publishing or subscribing, prefix all topics with a mountpoint string. + ## The prefixed string will be removed from the topic name when the message + ## is delivered to the subscriber. The mountpoint is a way that users can use + ## to implement isolation of message routing between different listeners. + ## + ## For example if a clientA subscribes to "t" with `listeners.ws..mountpoint` + ## set to "some_tenant", then the client accually subscribes to the topic + ## "some_tenant/t". Similarly if another clientB (connected to the same listener + ## with the clientA) send a message to topic "t", the message is accually route + ## to all the clients subscribed "some_tenant/t", so clientA will receive the + ## message, with topic name "t". + ## + ## Set to "" to disable the feature. + ## + ## Variables in mountpoint string: + ## - ${clientid}: clientid + ## - ${username}: username + ## + ## @doc listeners.ws..mountpoint + ## ValueType: String + ## Default: "" + mountpoint = "" + + ## TCP options + ## See ${example_common_tcp_options} for more information + tcp.backlog = 1024 + tcp.buffer = 4KB + + ## Websocket options + ## See ${example_common_websocket_options} for more information + websocket.idle_timeout = 86400s +} + +## MQTT/WSS - WebSocket Secure Listeners for MQTT Protocol +## syntax: listeners.wss. +## example: listeners.wss.my_wss_listener +listeners.wss.default { + ## The IP address and port that the listener will bind. + ## + ## @doc listeners.wss..bind + ## ValueType: IPAddress | Port | IPAddrPort + ## Required: true + ## Examples: 8084, 127.0.0.1:8084, ::1:8084 + bind = "0.0.0.0:8084" + + ## The configuration zone this listener is using. + ## If not set, the global configs are used for this listener. + ## + ## See `zones.` for more details. + ## + ## @doc listeners.wss..zone + ## ValueType: String + ## Required: false + #zone = default + + ## The size of the acceptor pool for this listener. + ## + ## @doc listeners.wss..acceptors + ## ValueType: Number + ## Default: 16 + acceptors = 16 + + ## Maximum number of concurrent connections. + ## + ## @doc listeners.wss..max_connections + ## ValueType: Number | infinity + ## Default: infinity + max_connections = 512000 + + ## The access control rules for this listener. + ## + ## See: https://github.com/emqtt/esockd#allowdeny + ## + ## @doc listeners.wss..access_rules + ## ValueType: Array + ## Default: [] + ## Examples: + ## access_rules: [ + ## "deny 192.168.0.0/24", + ## "all all" + ## ] + access_rules = [ + "allow all" + ] + + ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed + ## behind HAProxy or Nginx. + ## + ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ + ## + ## @doc listeners.wss..proxy_protocol + ## ValueType: Boolean + ## Default: true + proxy_protocol = false + + ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection + ## if no proxy protocol packet received within the timeout. + ## + ## @doc listeners.wss..proxy_protocol_timeout + ## ValueType: Duration + ## Default: 3s + proxy_protocol_timeout = 3s + + ## When publishing or subscribing, prefix all topics with a mountpoint string. + ## The prefixed string will be removed from the topic name when the message + ## is delivered to the subscriber. The mountpoint is a way that users can use + ## to implement isolation of message routing between different listeners. + ## + ## For example if a clientA subscribes to "t" with `listeners.wss..mountpoint` + ## set to "some_tenant", then the client accually subscribes to the topic + ## "some_tenant/t". Similarly if another clientB (connected to the same listener + ## with the clientA) send a message to topic "t", the message is accually route + ## to all the clients subscribed "some_tenant/t", so clientA will receive the + ## message, with topic name "t". + ## + ## Set to "" to disable the feature. + ## + ## Variables in mountpoint string: + ## - ${clientid}: clientid + ## - ${username}: username + ## + ## @doc listeners.wss..mountpoint + ## ValueType: String + ## Default: "" + mountpoint = "" + + ## SSL options + ## See ${example_common_ssl_options} for more information + ssl.keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem" + ssl.certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem" + ssl.cacertfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem" + + ## TCP options + ## See ${example_common_tcp_options} for more information + tcp.backlog = 1024 + tcp.buffer = 4KB + + ## Websocket options + ## See ${example_common_websocket_options} for more information + websocket.idle_timeout = 86400s + +} + +## Enable per connection statistics. +## +## @doc stats.enable +## ValueType: Boolean +## Default: true +stats.enable = true + +authorization { + ## Behaviour after not matching a rule. + ## + ## @doc authorization.no_match + ## ValueType: allow | deny + ## Default: allow + no_match: allow + + ## The action when authorization check reject current operation + ## + ## @doc authorization.deny_action + ## ValueType: ignore | disconnect + ## Default: ignore + deny_action: ignore + + ## Whether to enable Authorization cache. + ## + ## If enabled, Authorization roles for each client will be cached in the memory + ## + ## @doc authorization.cache.enable + ## ValueType: Boolean + ## Default: true + cache.enable: true + + ## The maximum count of Authorization entries can be cached for a client. + ## + ## @doc authorization.cache.max_size + ## ValueType: Integer + ## Range: [0, 1048576] + ## Default: 32 + cache.max_size: 32 + + ## The time after which an Authorization cache entry will be deleted + ## + ## @doc authorization.cache.ttl + ## ValueType: Duration + ## Default: 1m + cache.ttl: 1m +} + +mqtt { + ## How long time the MQTT connection will be disconnected if the + ## TCP connection is established but MQTT CONNECT has not been + ## received. + ## + ## @doc mqtt.idle_timeout + ## ValueType: Duration + ## Default: 15s + idle_timeout = 15s + + ## Maximum MQTT packet size allowed. + ## + ## @doc mqtt.max_packet_size + ## ValueType: Bytes + ## Default: 1MB + max_packet_size = 1MB + + ## Maximum length of MQTT clientId allowed. + ## + ## @doc mqtt.max_clientid_len + ## ValueType: Integer + ## Range: [23, 65535] + ## Default: 65535 + max_clientid_len = 65535 + + ## Maximum topic levels allowed. + ## + ## @doc mqtt.max_topic_levels + ## ValueType: Integer + ## Range: [1, 65535] + ## Default: 128 + ## Depth so big may lead to subscribing performance issues + max_topic_levels = 128 + + ## Maximum QoS allowed. + ## + ## @doc mqtt.max_qos_allowed + ## ValueType: 0 | 1 | 2 + ## Default: 2 + max_qos_allowed = 2 + + ## Maximum Topic Alias, 0 means no topic alias supported. + ## + ## @doc mqtt.max_topic_alias + ## ValueType: Integer + ## Range: [0, 65535] + ## Default: 65535 + max_topic_alias = 65535 + + ## Whether the Server supports MQTT retained messages. + ## + ## @doc mqtt.retain_available + ## ValueType: Boolean + ## Default: true + retain_available = true + + ## Whether the Server supports MQTT Wildcard Subscriptions + ## + ## @doc mqtt.wildcard_subscription + ## ValueType: Boolean + ## Default: true + wildcard_subscription = true + + ## Whether the Server supports MQTT Shared Subscriptions. + ## + ## @doc mqtt.shared_subscription + ## ValueType: Boolean + ## Default: true + shared_subscription = true + + ## Whether to ignore loop delivery of messages.(for mqtt v3.1.1) + ## + ## @doc mqtt.ignore_loop_deliver + ## ValueType: Boolean + ## Default: false + ignore_loop_deliver = false + + ## Whether to parse the MQTT frame in strict mode + ## + ## @doc mqtt.strict_mode + ## ValueType: Boolean + ## Default: false + strict_mode = false + + ## Specify the response information returned to the client + ## + ## This feature is disabled if is set to "" + ## + ## @doc mqtt.response_information + ## ValueType: String + ## Default: "" + response_information = "" + + ## Server Keep Alive of MQTT 5.0 + ## + ## @doc mqtt.server_keepalive + ## ValueType: Number | disabled + ## Default: disabled + server_keepalive = disabled + + ## The backoff for MQTT keepalive timeout. The broker will kick a connection out + ## until 'Keepalive * backoff * 2' timeout. + ## + ## @doc mqtt.keepalive_backoff + ## ValueType: Float + ## Range: (0.5, 1] + ## Default: 0.75 + keepalive_backoff = 0.75 + + ## Maximum number of subscriptions allowed. + ## + ## @doc mqtt.max_subscriptions + ## ValueType: Integer | infinity + ## Range: [1, infinity) + ## Default: infinity + max_subscriptions = infinity + + ## Force to upgrade QoS according to subscription. + ## + ## @doc mqtt.upgrade_qos + ## ValueType: Boolean + ## Default: false + upgrade_qos = false + + ## Maximum size of the Inflight Window storing QoS1/2 messages delivered but unacked. + ## + ## @doc mqtt.max_inflight + ## ValueType: Integer + ## Range: [1, 65535] + ## Default: 32 + max_inflight = 32 + + ## Retry interval for QoS1/2 message delivering. + ## + ## @doc mqtt.retry_interval + ## ValueType: Duration + ## Default: 30s + retry_interval = 30s + + ## Maximum QoS2 packets (Client -> Broker) awaiting PUBREL. + ## + ## @doc mqtt.max_awaiting_rel + ## ValueType: Integer | infinity + ## Range: [1, infinity) + ## Default: 100 + max_awaiting_rel = 100 + + ## The QoS2 messages (Client -> Broker) will be dropped if awaiting PUBREL timeout. + ## + ## @doc mqtt.await_rel_timeout + ## ValueType: Duration + ## Default: 300s + await_rel_timeout = 300s + + ## Default session expiry interval for MQTT V3.1.1 connections. + ## + ## @doc mqtt.session_expiry_interval + ## ValueType: Duration + ## Default: 2h + session_expiry_interval = 2h + + ## Maximum queue length. Enqueued messages when persistent client disconnected, + ## or inflight window is full. + ## + ## @doc mqtt.max_mqueue_len + ## ValueType: Integer | infinity + ## Range: [0, infinity) + ## Default: 1000 + max_mqueue_len = 1000 + + ## Topic priorities. + ## + ## There's no priority table by default, hence all messages + ## are treated equal. + ## + ## Priority number [1-255] + ## + ## NOTE: comma and equal signs are not allowed for priority topic names + ## NOTE: Messages for topics not in the priority table are treated as + ## either highest or lowest priority depending on the configured + ## value for mqtt.mqueue_default_priority + ## + ## @doc mqtt.mqueue_priorities + ## ValueType: Map | disabled + ## Examples: + ## To configure "topic/1" > "topic/2": + ## mqueue_priorities: {"topic/1": 10, "topic/2": 8} + ## Default: disabled + mqueue_priorities = disabled + + ## Default to highest priority for topics not matching priority table + ## + ## @doc mqtt.mqueue_default_priority + ## ValueType: highest | lowest + ## Default: lowest + mqueue_default_priority = lowest + + ## Whether to enqueue QoS0 messages. + ## + ## @doc mqtt.mqueue_store_qos0 + ## ValueType: Boolean + ## Default: true + mqueue_store_qos0 = true + + ## Whether use username replace client id + ## + ## @doc mqtt.use_username_as_clientid + ## ValueType: Boolean + ## Default: false + use_username_as_clientid = false + + ## Use the CN, DN or CRT field from the client certificate as a username. + ## Only works for SSL connection. + ## + ## @doc mqtt.peer_cert_as_username + ## ValueType: cn | dn | crt | disabled + ## Default: disabled + peer_cert_as_username = disabled + + ## Use the CN, DN or CRT field from the client certificate as a clientid. + ## Only works for SSL connection. + ## + ## @doc mqtt.peer_cert_as_clientid + ## ValueType: cn | dn | crt | disabled + ## Default: disabled + peer_cert_as_clientid = disabled +} + +flapping_detect { + ## Enable Flapping Detection. + ## + ## This config controls the allowed maximum number of CONNECT received + ## from the same clientid in a time frame defined by `window_time`. + ## After the limit is reached, successive CONNECT requests are forbidden + ## (banned) until the end of the time period defined by `ban_time`. + ## + ## @doc flapping_detect.enable + ## ValueType: Boolean + ## Default: true + enable = false + + ## The max disconnect allowed of a MQTT Client in `window_time` + ## + ## @doc flapping_detect.max_count + ## ValueType: Integer + ## Default: 15 + max_count = 15 + + ## The time window for flapping detect + ## + ## @doc flapping_detect.window_time + ## ValueType: Duration + ## Default: 1m + window_time = 1m + + ## How long the clientid will be banned + ## + ## @doc flapping_detect.ban_time + ## ValueType: Duration + ## Default: 5m + ban_time = 5m + +} + +force_shutdown { + ## Enable force_shutdown + ## + ## @doc force_shutdown.enable + ## ValueType: Boolean + ## Default: true + enable = true + + ## Max message queue length + ## @doc force_shutdown.max_message_queue_len + ## ValueType: Integer + ## Range: (0, infinity) + ## Default: 1000 + max_message_queue_len = 1000 + + ## Total heap size + ## + ## @doc force_shutdown.max_heap_size + ## ValueType: Size + ## Default: 32MB + max_heap_size = 32MB +} + +overload_protection { + ## React on system overload or not + ## @doc overload_protection.enable + ## ValueType: Boolean + ## Default: false + enable = false + + ## Backoff delay in ms + ## @doc overload_protection.backoff_delay + ## ValueType: Integer + ## Range: (0, infinity) + ## Default: 1 + backoff_delay = 1 + + ## Backoff GC enabled + ## @doc overload_protection.backoff_gc + ## ValueType: Boolean + ## Default: false + backoff_gc = false + + ## Backoff hibernation enabled + ## @doc overload_protection.backoff_hibernation + ## ValueType: Boolean + ## Default: true + backoff_hibernation = true + + ## Backoff hibernation enabled + ## @doc overload_protection.backoff_hibernation + ## ValueType: Boolean + ## Default: true + backoff_new_conn = true +} + +force_gc { + ## Force the MQTT connection process GC after this number of + ## messages or bytes passed through. + ## + ## @doc force_gc.enable + ## ValueType: Boolean + ## Default: true + enable = true + + ## GC the process after how many messages received + ## @doc force_gc.max_message_queue_len + ## ValueType: Integer + ## Range: (0, infinity) + ## Default: 16000 + count = 16000 + + ## GC the process after how much bytes passed through + ## + ## @doc force_gc.bytes + ## ValueType: Size + ## Default: 16MB + bytes = 16MB +} + +conn_congestion { + ## Whether to alarm the congested connections. + ## + ## Sometimes the mqtt connection (usually an MQTT subscriber) may + ## get "congested" because there're too many packets to sent. + ## The socket trys to buffer the packets until the buffer is + ## full. If more packets comes after that, the packets will be + ## "pending" in a queue and we consider the connection is + ## "congested". + ## + ## Enable this to send an alarm when there's any bytes pending in + ## the queue. You could set the `sndbuf` to a larger value if the + ## alarm is triggered too often. + ## + ## The name of the alarm is of format "conn_congestion//". + ## Where the is the client-id of the congested MQTT connection. + ## And the is the username or "unknown_user" of not provided by the client. + ## + ## @doc conn_congestion.enable_alarm + ## ValueType: Boolean + ## Default: true + enable_alarm = true + + ## Won't clear the congested alarm in how long time. + ## The alarm is cleared only when there're no pending bytes in + ## the queue, and also it has been `min_alarm_sustain_duration` + ## time since the last time we considered the connection is "congested". + ## + ## This is to avoid clearing and sending the alarm again too often. + ## + ## @doc conn_congestion.min_alarm_sustain_duration + ## ValueType: Duration + ## Default: 1m + min_alarm_sustain_duration = 1m +} + +rate_limit { + ## Maximum connections per second. + ## + ## @doc zones..max_conn_rate + ## ValueType: Number | infinity + ## Default: 1000 + ## Examples: + ## max_conn_rate: 1000 + max_conn_rate = 1000 + + ## Message limit for the a external MQTT connection. + ## + ## @doc rate_limit.conn_messages_in + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100 messages per 10 seconds. + ## conn_messages_in: "100,10s" + conn_messages_in = "100,10s" + + ## Limit the rate of receiving packets for a MQTT connection. + ## The rate is counted by bytes of packets per second. + ## + ## The connection won't accept more messages if the messages come + ## faster than the limit. + ## + ## @doc rate_limit.conn_bytes_in + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100KB incoming per 10 seconds. + ## conn_bytes_in: "100KB,10s" + ## + conn_bytes_in = "100KB,10s" +} + +quota { + ## Messages quota for the each of external MQTT connection. + ## This value consumed by the number of recipient on a message. + ## + ## @doc quota.conn_messages_routing + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 100 messaegs per 1s: + ## quota.conn_messages_routing: "100,1s" + conn_messages_routing = "100,1s" + + ## Messages quota for the all of external MQTT connections. + ## This value consumed by the number of recipient on a message. + ## + ## @doc quota.overall_messages_routing + ## ValueType: String | infinity + ## Default: infinity + ## Examples: 200000 messages per 1s: + ## quota.overall_messages_routing: "200000,1s" + ## + overall_messages_routing = "200000,1s" +} + +##================================================================== +## Zones +##================================================================== +## A zone contains a set of configurations for listeners. +## +## A zone can be used by a listener via `listener...zone`. +## +## The configs defined in zones will override the global configs with the same key. +## +## For example given the following config: +## +## ``` +## a { +## b: 1, c: 1 +## } +## +## zone.my_zone { +## a { +## b:2 +## } +## } +## ``` +## +## The global config "a" is overridden by the configs "a" inside the zone "my_zone". +## If there is a listener uses the zone "my_zone", the value of config "a" will be: +## `{b:2, c: 1}`. +## Note that although the default value of `a.c` is `0`, the global value is used. +## i.e. configs in the zone have no default values. To overridde `a.c` we must configure +## it explicitly in the zone. +## +## All the global configs that can be overridden in zones are: +## - `stats.*` +## - `mqtt.*` +## - `authorization.*` +## - `flapping_detect.*` +## - `force_shutdown.*` +## - `conn_congestion.*` +## - `rate_limit.*` +## - `quota.*` +## - `force_gc.*` +## +## syntax: zones. +## example: zones.my_zone +zones.default { + +} + +##================================================================== +## Broker +##================================================================== +broker { + ## System interval of publishing $SYS messages. + ## + ## @doc broker.sys_msg_interval + ## ValueType: Duration | disabled + ## Default: 1m + sys_msg_interval = 1m + + ## System heartbeat interval of publishing following heart beat message: + ## - "$SYS/brokers//uptime" + ## - "$SYS/brokers//datetime" + ## + ## @doc broker.sys_heartbeat_interval + ## ValueType: Duration + ## Default: 30s | disabled + sys_heartbeat_interval = 30s + + ## Session locking strategy in a cluster. + ## + ## @doc broker.session_locking_strategy + ## ValueType: local | one | quorum | all + ## - local: only lock the session locally on the current node + ## - one: select only one remove node to lock the session + ## - quorum: select some nodes to lock the session + ## - all: lock the session on all of the nodes in the cluster + ## Default: quorum + session_locking_strategy = quorum + + ## Dispatch strategy for shared subscription + ## + ## @doc broker.shared_subscription_strategy + ## ValueType: random | round_robin | sticky | hash + ## - random: dispatch the message to a random selected subscriber + ## - round_robin: select the subscribers in a round-robin manner + ## - sticky: always use the last selected subscriber to dispatch, + ## until the susbcriber disconnected. + ## - hash: select the subscribers by the hash of clientIds + ## Default: round_robin + shared_subscription_strategy = round_robin + + ## Enable/disable shared dispatch acknowledgement for QoS1 and QoS2 messages + ## This should allow messages to be dispatched to a different subscriber in + ## the group in case the picked (based on shared_subscription_strategy) one # is offline + ## + ## @doc broker.shared_dispatch_ack_enabled + ## ValueType: Boolean + ## Default: false + shared_dispatch_ack_enabled = false + + ## Enable batch clean for deleted routes. + ## + ## @doc broker.route_batch_clean + ## ValueType: Boolean + ## Default: true + route_batch_clean = true + + ## Performance toggle for subscribe/unsubscribe wildcard topic. + ## Change this toggle only when there are many wildcard topics. + ## + ## NOTE: when changing from/to 'global' lock, it requires all + ## nodes in the cluster to be stopped before the change. + ## + ## @doc broker.perf.route_lock_type + ## ValueType: key | tab | global + ## - key: mnesia translational updates with per-key locks. recommended for single node setup. + ## - tab: mnesia translational updates with table lock. recommended for multi-nodes setup. + ## - global: global lock protected updates. recommended for larger cluster. + ## Default: key + perf.route_lock_type = key + + ## Enable trie path compaction. + ## Enabling it significantly improves wildcard topic subscribe + ## rate, if wildcard topics have unique prefixes like: + ## 'sensor//+/', where ID is unique per subscriber. + ## + ## Topic match performance (when publishing) may degrade if messages + ## are mostly published to topics with large number of levels. + ## + ## NOTE: This is a cluster-wide configuration. + ## It requires all nodes to be stopped before changing it. + ## + ## @doc broker.perf.trie_compaction + ## ValueType: Boolean + ## Default: true + perf.trie_compaction = true +} + +##================================================================== +## System Monitor +##================================================================== +sysmon { + ## The time interval for the periodic process limit check + ## + ## @doc sysmon.vm.process_check_interval + ## ValueType: Duration + ## Default: 30s + vm.process_check_interval = 30s + + ## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is set. + ## + ## @doc sysmon.vm.process_high_watermark + ## ValueType: Percentage + ## Default: 80% + vm.process_high_watermark = 80% + + ## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is clear. + ## + ## @doc sysmon.vm.process_low_watermark + ## ValueType: Percentage + ## Default: 60% + vm.process_low_watermark = 60% + + ## Enable Long GC monitoring. + ## Notice: don't enable the monitor in production for: + ## https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421 + ## + ## @doc sysmon.vm.long_gc + ## ValueType: Duration | disabled + ## Default: disabled + vm.long_gc = disabled + + ## Enable Long Schedule(ms) monitoring. + ## + ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 + ## + ## @doc sysmon.vm.long_schedule + ## ValueType: Duration | disabled + ## Default: disabled + vm.long_schedule = 240ms + + ## Enable Large Heap monitoring. + ## + ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 + ## + ## @doc sysmon.vm.large_heap + ## ValueType: Size | disabled + ## Default: 32MB + vm.large_heap = 32MB + + ## Enable Busy Port monitoring. + ## + ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 + ## + ## @doc sysmon.vm.busy_port + ## ValueType: Boolean + ## Default: true + vm.busy_port = true + + ## Enable Busy Dist Port monitoring. + ## + ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 + ## + ## @doc sysmon.vm.busy_dist_port + ## ValueType: Boolean + ## Default: true + vm.busy_dist_port = true + + ## The time interval for the periodic cpu check + ## + ## @doc sysmon.os.cpu_check_interval + ## ValueType: Duration + ## Default: 60s + os.cpu_check_interval = 60s + + ## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is set. + ## + ## @doc sysmon.os.cpu_high_watermark + ## ValueType: Percentage + ## Default: 80% + os.cpu_high_watermark = 80% + + ## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is clear. + ## + ## @doc sysmon.os.cpu_low_watermark + ## ValueType: Percentage + ## Default: 60% + os.cpu_low_watermark = 60% + + ## The time interval for the periodic memory check + ## + ## @doc sysmon.os.mem_check_interval + ## ValueType: Duration | disabled + ## Default: 60s + os.mem_check_interval = 60s + + ## The threshold, as percentage of system memory, for how much system memory can be allocated before the corresponding alarm is set. + ## + ## @doc sysmon.os.sysmem_high_watermark + ## ValueType: Percentage + ## Default: 70% + os.sysmem_high_watermark = 70% + + ## The threshold, as percentage of system memory, for how much system memory can be allocated by one Erlang process before the corresponding alarm is set. + ## + ## @doc sysmon.os.procmem_high_watermark + ## ValueType: Percentage + ## Default: 5% + os.procmem_high_watermark = 5% +} + +##================================================================== +## Alarm +##================================================================== +alarm { + ## Specifies the actions to take when an alarm is activated + ## + ## @doc alarm.actions + ## ValueType: Array + ## Default: [log, publish] + actions = [log, publish] + + ## The maximum number of deactivated alarms + ## + ## @doc alarm.size_limit + ## ValueType: Integer + ## Default: 1000 + size_limit = 1000 + + ## Validity Period of deactivated alarms + ## + ## @doc alarm.validity_period + ## ValueType: Duration + ## Default: 24h + validity_period = 24h +} + +## Config references for listeners + +## Socket options for TCP connections +## See: http://erlang.org/doc/man/inet.html +example_common_tcp_options { + ## Specify the {active, N} option for this Socket. + ## + ## See: https://erlang.org/doc/man/inet.html#setopts-2 + ## + ## @doc listeners..tcp.active_n + ## ValueType: Number + ## Default: 100 + tcp.active_n = 100 + + ## TCP backlog defines the maximum length that the queue of + ## pending connections can grow to. + ## + ## @doc listeners..tcp.backlog + ## ValueType: Number + ## Range: [0, 1048576] + ## Default: 1024 + tcp.backlog = 1024 + + ## The TCP send timeout for the connections. + ## + ## @doc listeners..tcp.send_timeout + ## ValueType: Duration + ## Default: 15s + tcp.send_timeout = 15s + + ## Close the connection if send timeout. + ## + ## @doc listeners..tcp.send_timeout_close + ## ValueType: Boolean + ## Default: true + tcp.send_timeout_close = true + + ## The TCP receive buffer(os kernel) for the connections. + ## + ## @doc listeners..tcp.recbuf + ## ValueType: Size + ## Default: notset + #tcp.recbuf: 2KB + + ## The TCP send buffer(os kernel) for the connections. + ## + ## @doc listeners..tcp.sndbuf + ## ValueType: Size + ## Default: notset + #tcp.sndbuf: 4KB + + ## The size of the user-level software buffer used by the driver. + ## + ## @doc listeners..tcp.buffer + ## ValueType: Size + ## Default: notset + #tcp.buffer: 4KB + + ## The socket is set to a busy state when the amount of data queued internally + ## by the ERTS socket implementation reaches this limit. + ## + ## @doc listeners..tcp.high_watermark + ## ValueType: Size + ## Default: 1MB + tcp.high_watermark = 1MB + + ## The TCP_NODELAY flag for the connections. + ## + ## @doc listeners..tcp.nodelay + ## ValueType: Boolean + ## Default: false + tcp.nodelay = false + + ## The SO_REUSEADDR flag for the connections. + ## + ## @doc listeners..tcp.reuseaddr + ## ValueType: Boolean + ## Default: true + tcp.reuseaddr = true +} + +## Socket options for SSL connections +## See: http://erlang.org/doc/man/ssl.html +example_common_ssl_options { + + ## A performance optimization setting, it allows clients to reuse + ## pre-existing sessions, instead of initializing new ones. + ## Read more about it here. + ## + ## @doc listeners..ssl.reuse_sessions + ## ValueType: Boolean + ## Default: true + ssl.reuse_sessions = true + + ## SSL parameter renegotiation is a feature that allows a client and a server + ## to renegotiate the parameters of the SSL connection on the fly. + ## RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, + ## you drop support for the insecure renegotiation, prone to MitM attacks. + ## + ## @doc listeners..ssl.secure_renegotiate + ## ValueType: Boolean + ## Default: true + ssl.secure_renegotiate = true + + ## In protocols that support client-initiated renegotiation, + ## the cost of resources of such an operation is higher for the server than the client. + ## This can act as a vector for denial of service attacks. + ## The SSL application already takes measures to counter-act such attempts, + ## but client-initiated renegotiation can be strictly disabled by setting this option to false. + ## The default value is true. Note that disabling renegotiation can result in + ## long-lived connections becoming unusable due to limits on + ## the number of messages the underlying cipher suite can encipher. + ssl.client_renegotiation = true + + ## An important security setting, it forces the cipher to be set based + ## on the server-specified order instead of the client-specified order, + ## hence enforcing the (usually more properly configured) security + ## ordering of the server administrator. + ## + ## @doc listeners..ssl.honor_cipher_order + ## ValueType: Boolean + ## Default: true + ssl.honor_cipher_order = true + + # ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + # TLS 1.3: "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256" + # TLS 1-1.2 "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" + # PSK: "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" + # NOTE: If PSK cipher-suites are intended, tlsv1.3 should not be enabled in 'versions' config + # NOTE: by default, ALL ciphers are enabled + # ssl.ciphers = "" + + ## TLS Handshake timeout. + ## + ## @doc listeners..ssl.handshake_timeout + ## ValueType: Duration + ## Default: 15s + ssl.handshake_timeout = 15s + + ## Maximum number of non-self-issued intermediate certificates that + ## can follow the peer certificate in a valid certification path. + ## + ## @doc listeners..ssl.depth + ## ValueType: Integer + ## Default: 10 + ssl.depth = 10 + + ## Path to the file containing the user's private PEM-encoded key. + ## + ## @doc listeners..ssl.keyfile + ## ValueType: File + ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem" + ssl.keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem" + + ## Path to a file containing the user certificate. + ## + ## @doc listeners..ssl.certfile + ## ValueType: File + ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem" + ssl.certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem" + + ## Path to the file containing PEM-encoded CA certificates. The CA certificates + ## are used during server authentication and when building the client certificate chain. + ## + ## @doc listeners..ssl.cacertfile + ## ValueType: File + ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem" + ssl.cacertfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem" + + ## Maximum number of non-self-issued intermediate certificates that + ## can follow the peer certificate in a valid certification path. + ## + ## @doc listeners..ssl.depth + ## ValueType: Number + ## Default: 10 + ssl.depth = 10 + + ## String containing the user's password. Only used if the private keyfile + ## is password-protected. + ## + ## See: listener.ssl.$name.key_password + ## + ## @doc listeners..ssl.depth + ## ValueType: String + ## Default: "" + #ssl.key_password: "" + + ## The Ephemeral Diffie-Helman key exchange is a very effective way of + ## ensuring Forward Secrecy by exchanging a set of keys that never hit + ## the wire. Since the DH key is effectively signed by the private key, + ## it needs to be at least as strong as the private key. In addition, + ## the default DH groups that most of the OpenSSL installations have + ## are only a handful (since they are distributed with the OpenSSL + ## package that has been built for the operating system it’s running on) + ## and hence predictable (not to mention, 1024 bits only). + ## In order to escape this situation, first we need to generate a fresh, + ## strong DH group, store it in a file and then use the option above, + ## to force our SSL application to use the new DH group. Fortunately, + ## OpenSSL provides us with a tool to do that. Simply run: + ## openssl dhparam -out dh-params.pem 2048 + ## + ## @doc listeners..ssl.dhfile + ## ValueType: File + ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/dh-params.pem" + #ssl.dhfile: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/dh-params.pem" + + ## A server only does x509-path validation in mode verify_peer, + ## as it then sends a certificate request to the client (this + ## message is not sent if the verify option is verify_none). + ## You can then also want to specify option fail_if_no_peer_cert. + ## More information at: http://erlang.org/doc/man/ssl.html + ## + ## @doc listeners..ssl.verify + ## ValueType: verify_peer | verify_none + ## Default: verify_none + ssl.verify = verify_none + + ## Used together with {verify, verify_peer} by an SSL server. If set to true, + ## the server fails if the client does not have a certificate to send, that is, + ## sends an empty certificate. + ## + ## @doc listeners..ssl.fail_if_no_peer_cert + ## ValueType: Boolean + ## Default: true + ssl.fail_if_no_peer_cert = false + +} + +## Socket options for websocket connections +example_common_websocket_options { + ## The path of WebSocket MQTT endpoint + ## + ## @doc listeners..websocket.mqtt_path + ## ValueType: Path + ## Default: "/mqtt" + websocket.mqtt_path = "/mqtt" + + ## Whether a WebSocket message is allowed to contain multiple MQTT packets + ## + ## @doc listeners..websocket.mqtt_piggyback + ## ValueType: single | multiple + ## Default: multiple + websocket.mqtt_piggyback = multiple + + ## The compress flag for external WebSocket connections. + ## + ## If this Value is set true,the websocket message would be compressed + ## + ## @doc listeners..websocket.compress + ## ValueType: Boolean + ## Default: false + websocket.compress = false + + ## The idle timeout for external WebSocket connections. + ## + ## @doc listeners..websocket.idle_timeout + ## ValueType: Duration | infinity + ## Default: infinity + websocket.idle_timeout = infinity + + ## The max frame size for external WebSocket connections. + ## + ## @doc listeners..websocket.max_frame_size + ## ValueType: Size + ## Default: infinity + websocket.max_frame_size = infinity + + ## If set to true, the server fails if the client does not + ## have a Sec-WebSocket-Protocol to send. + ## Set to false for WeChat MiniApp. + ## + ## @doc listeners..websocket.fail_if_no_subprotocol + ## ValueType: Boolean + ## Default: true + websocket.fail_if_no_subprotocol = true + + ## Supported subprotocols + ## + ## @doc listeners..websocket.supported_subprotocols + ## ValueType: String + ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 + websocket.supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" + + ## Enable origin check in header for websocket connection + ## + ## @doc listeners..websocket.check_origin_enable + ## ValueType: Boolean + ## Default: false + websocket.check_origin_enable = false + + ## Allow origin to be absent in header in websocket connection + ## when check_origin_enable is true + ## + ## @doc listeners..websocket.allow_origin_absence + ## ValueType: Boolean + ## Default: true + websocket.allow_origin_absence = true + + ## Comma separated list of allowed origin in header for websocket connection + ## + ## @doc listeners..websocket.check_origins + ## ValueType: String + ## Examples: + ## local http dashboard url + ## check_origins: "http://localhost:18083, http://127.0.0.1:18083" + ## Default: "" + websocket.check_origins = "http://localhost:18083, http://127.0.0.1:18083" + + ## Specify which HTTP header for real source IP if the EMQ X cluster is + ## deployed behind NGINX or HAProxy. + ## + ## @doc listeners..websocket.proxy_address_header + ## ValueType: String + ## Default: X-Forwarded-For + websocket.proxy_address_header = X-Forwarded-For + + ## Specify which HTTP header for real source port if the EMQ X cluster is + ## deployed behind NGINX or HAProxy. + ## + ## @doc listeners..websocket.proxy_port_header + ## ValueType: String + ## Default: X-Forwarded-Port + websocket.proxy_port_header = X-Forwarded-Port + + websocket.deflate_opts { + ## The level of deflate options for external WebSocket connections. + ## + ## @doc listeners..websocket.deflate_opts.level + ## ValueType: none | default | best_compression | best_speed + ## Default: default + level = default + + ## The mem_level of deflate options for external WebSocket connections. + ## + ## @doc listeners..websocket.deflate_opts.mem_level + ## ValueType: Integer + ## Range: [1,9] + ## Default: 8 + mem_level = 8 + + ## The strategy of deflate options for external WebSocket connections. + ## + ## @doc listeners..websocket.deflate_opts.strategy + ## ValueType: default | filtered | huffman_only | rle + ## Default: default + strategy = default + + ## The deflate option for external WebSocket connections. + ## + ## @doc listeners..websocket.deflate_opts.server_context_takeover + ## ValueType: takeover | no_takeover + ## Default: takeover + server_context_takeover = takeover + + ## The deflate option for external WebSocket connections. + ## + ## @doc listeners..websocket.deflate_opts.client_context_takeover + ## ValueType: takeover | no_takeover + ## Default: takeover + client_context_takeover = takeover + + ## The deflate options for external WebSocket connections. + ## + ## + ## @doc listeners..websocket.deflate_opts.server_max_window_bits + ## ValueType: Integer + ## Range: [8,15] + ## Default: 15 + server_max_window_bits = 15 + + ## The deflate options for external WebSocket connections. + ## + ## @doc listeners..websocket.deflate_opts.client_max_window_bits + ## ValueType: Integer + ## Range: [8,15] + ## Default: 15 + client_max_window_bits = 15 + } +} + +persistent_session_store { + ## Enable/disable internal persistent session store. + ## + ## @doc persistent_session_store.enabled + ## ValueType: Boolean + ## Default: false + enabled = false + + ## How long are undelivered messages retained in the store + ## + ## @doc persistent_session_store.max_retain_undelivered + ## ValueType: Duration + ## Default: 1h + max_retain_undelivered = 1h + + ## The time interval in which to try to run garbage collection of persistent session messages + ## + ## @doc persistent_session_store.message_gc_interval + ## ValueType: Duration + ## Default: 1h + message_gc_interval = 1h + + ## The time interval in which to try to run garbage collection of persistent session transient data + ## + ## @doc persistent_session_store.session_message_gc_interval + ## ValueType: Duration + ## Default: 1m + session_message_gc_interval = 1m +} diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 7bca3ca3c..67542a2bf 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -112,7 +112,6 @@ log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) -> case logger_config:get(ets:whereis(logger), Id) of {ok, #{module := Module} = HandlerConfig0} -> HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0), - io:format("~p~n", [{Module, Log, HandlerConfig}]), try Module:log(Log, HandlerConfig) catch C:R:S -> case logger:remove_handler(Id) of diff --git a/apps/emqx/test/emqx_trace_handler_SUITE.erl b/apps/emqx/test/emqx_trace_handler_SUITE.erl index 01d587028..f3ab7b5b8 100644 --- a/apps/emqx/test/emqx_trace_handler_SUITE.erl +++ b/apps/emqx/test/emqx_trace_handler_SUITE.erl @@ -50,15 +50,15 @@ end_per_testcase(_Case, _Config) -> t_trace_clientid(_Config) -> %% Start tracing %% add list clientid - ok = emqx_trace_handler:install(clientid, "client", debug, "tmp/client.log"), - ok = emqx_trace_handler:install(clientid, <<"client2">>, all, "tmp/client2.log"), - ok = emqx_trace_handler:install(clientid, <<"client3">>, all, "tmp/client3.log"), + ok = emqx_trace_handler:install("CLI-client1", clientid, "client", debug, "tmp/client.log"), + ok = emqx_trace_handler:install("CLI-client2", clientid, <<"client2">>, all, "tmp/client2.log"), + ok = emqx_trace_handler:install("CLI-client3", clientid, <<"client3">>, all, "tmp/client3.log"), {error, {handler_not_added, {file_error, ".", eisdir}}} = emqx_trace_handler:install(clientid, <<"client5">>, debug, "."), emqx_trace:check(), - ok = filesync(<<"client">>, clientid), - ok = filesync(<<"client2">>, clientid), - ok = filesync(<<"client3">>, clientid), + ok = filesync(<<"CLI-client1">>, clientid), + ok = filesync(<<"CLI-client2">>, clientid), + ok = filesync(<<"CLI-client3">>, clientid), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/client.log")), @@ -66,11 +66,11 @@ t_trace_clientid(_Config) -> ?assert(filelib:is_regular("tmp/client3.log")), %% Get current traces - ?assertMatch([#{type := clientid, filter := "client", name := <<"client">>, + ?assertMatch([#{type := clientid, filter := "client", name := <<"CLI-client1">>, level := debug, dst := "tmp/client.log"}, - #{type := clientid, filter := "client2", name := <<"client2">> + #{type := clientid, filter := "client2", name := <<"CLI-client2">> , level := debug, dst := "tmp/client2.log"}, - #{type := clientid, filter := "client3", name := <<"client3">>, + #{type := clientid, filter := "client3", name := <<"CLI-client3">>, level := debug, dst := "tmp/client3.log"} ], emqx_trace_handler:running()), @@ -79,9 +79,9 @@ t_trace_clientid(_Config) -> emqtt:connect(T), emqtt:publish(T, <<"a/b/c">>, <<"hi">>), emqtt:ping(T), - ok = filesync(<<"client">>, clientid), - ok = filesync(<<"client2">>, clientid), - ok = filesync(<<"client3">>, clientid), + ok = filesync(<<"CLI-client1">>, clientid), + ok = filesync(<<"CLI-client2">>, clientid), + ok = filesync(<<"CLI-client3">>, clientid), %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log". {ok, Bin} = file:read_file("tmp/client.log"), @@ -92,24 +92,24 @@ t_trace_clientid(_Config) -> ?assert(filelib:file_size("tmp/client2.log") == 0), %% Stop tracing - ok = emqx_trace_handler:uninstall(clientid, <<"client">>), - ok = emqx_trace_handler:uninstall(clientid, <<"client2">>), - ok = emqx_trace_handler:uninstall(clientid, <<"client3">>), + ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client1">>), + ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client2">>), + ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client3">>), emqtt:disconnect(T), ?assertEqual([], emqx_trace_handler:running()). t_trace_clientid_utf8(_) -> Utf8Id = <<"client 漢字編碼"/utf8>>, - ok = emqx_trace_handler:install(clientid, Utf8Id, debug, "tmp/client-utf8.log"), + ok = emqx_trace_handler:install("CLI-UTF8", clientid, Utf8Id, debug, "tmp/client-utf8.log"), emqx_trace:check(), {ok, T} = emqtt:start_link([{clientid, Utf8Id}]), emqtt:connect(T), [begin emqtt:publish(T, <<"a/b/c">>, <<"hi">>) end|| _ <- lists:seq(1, 10)], emqtt:ping(T), - ok = filesync(Utf8Id, clientid), - ok = emqx_trace_handler:uninstall(clientid, Utf8Id), + ok = filesync("CLI-UTF8", clientid), + ok = emqx_trace_handler:uninstall(clientid, "CLI-UTF8"), emqtt:disconnect(T), ?assertEqual([], emqx_trace_handler:running()), ok. @@ -119,11 +119,11 @@ t_trace_topic(_Config) -> emqtt:connect(T), %% Start tracing - ok = emqx_trace_handler:install(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"), - ok = emqx_trace_handler:install(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"), + ok = emqx_trace_handler:install("CLI-TOPIC-1", topic, <<"x/#">>, all, "tmp/topic_trace_x.log"), + ok = emqx_trace_handler:install("CLI-TOPIC-2", topic, <<"y/#">>, all, "tmp/topic_trace_y.log"), emqx_trace:check(), - ok = filesync(<<"x/#">>, topic), - ok = filesync(<<"y/#">>, topic), + ok = filesync("CLI-TOPIC-1", topic), + ok = filesync("CLI-TOPIC-2", topic), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/topic_trace_x.log")), @@ -131,9 +131,9 @@ t_trace_topic(_Config) -> %% Get current traces ?assertMatch([#{type := topic, filter := <<"x/#">>, - level := debug, dst := "tmp/topic_trace_x.log", name := <<"x/#">>}, + level := debug, dst := "tmp/topic_trace_x.log", name := <<"CLI-TOPIC-1">>}, #{type := topic, filter := <<"y/#">>, - name := <<"y/#">>, level := debug, dst := "tmp/topic_trace_y.log"} + name := <<"CLI-TOPIC-2">>, level := debug, dst := "tmp/topic_trace_y.log"} ], emqx_trace_handler:running()), @@ -142,8 +142,8 @@ t_trace_topic(_Config) -> emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), emqtt:subscribe(T, <<"x/y/z">>), emqtt:unsubscribe(T, <<"x/y/z">>), - ok = filesync(<<"x/#">>, topic), - ok = filesync(<<"y/#">>, topic), + ok = filesync("CLI-TOPIC-1", topic), + ok = filesync("CLI-TOPIC-2", topic), {ok, Bin} = file:read_file("tmp/topic_trace_x.log"), ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), @@ -154,8 +154,8 @@ t_trace_topic(_Config) -> ?assert(filelib:file_size("tmp/topic_trace_y.log") =:= 0), %% Stop tracing - ok = emqx_trace_handler:uninstall(topic, <<"x/#">>), - ok = emqx_trace_handler:uninstall(topic, <<"y/#">>), + ok = emqx_trace_handler:uninstall(topic, <<"CLI-TOPIC-1">>), + ok = emqx_trace_handler:uninstall(topic, <<"CLI-TOPIC-2">>), {error, _Reason} = emqx_trace_handler:uninstall(topic, <<"z/#">>), ?assertEqual([], emqx_trace_handler:running()), emqtt:disconnect(T). @@ -165,11 +165,12 @@ t_trace_ip_address(_Config) -> emqtt:connect(T), %% Start tracing - ok = emqx_trace_handler:install(ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"), - ok = emqx_trace_handler:install(ip_address, "192.168.1.1", all, "tmp/ip_trace_y.log"), + ok = emqx_trace_handler:install("CLI-IP-1", ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"), + ok = emqx_trace_handler:install("CLI-IP-2", ip_address, + "192.168.1.1", all, "tmp/ip_trace_y.log"), emqx_trace:check(), - ok = filesync(<<"127.0.0.1">>, ip_address), - ok = filesync(<<"192.168.1.1">>, ip_address), + ok = filesync(<<"CLI-IP-1">>, ip_address), + ok = filesync(<<"CLI-IP-2">>, ip_address), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/ip_trace_x.log")), @@ -177,10 +178,10 @@ t_trace_ip_address(_Config) -> %% Get current traces ?assertMatch([#{type := ip_address, filter := "127.0.0.1", - name := <<"127.0.0.1">>, + name := <<"CLI-IP-1">>, level := debug, dst := "tmp/ip_trace_x.log"}, #{type := ip_address, filter := "192.168.1.1", - name := <<"192.168.1.1">>, + name := <<"CLI-IP-2">>, level := debug, dst := "tmp/ip_trace_y.log"} ], emqx_trace_handler:running()), @@ -190,8 +191,8 @@ t_trace_ip_address(_Config) -> emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), emqtt:subscribe(T, <<"x/y/z">>), emqtt:unsubscribe(T, <<"x/y/z">>), - ok = filesync(<<"127.0.0.1">>, ip_address), - ok = filesync(<<"192.168.1.1">>, ip_address), + ok = filesync(<<"CLI-IP-1">>, ip_address), + ok = filesync(<<"CLI-IP-2">>, ip_address), {ok, Bin} = file:read_file("tmp/ip_trace_x.log"), ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), @@ -202,8 +203,8 @@ t_trace_ip_address(_Config) -> ?assert(filelib:file_size("tmp/ip_trace_y.log") =:= 0), %% Stop tracing - ok = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.1">>), - ok = emqx_trace_handler:uninstall(ip_address, <<"192.168.1.1">>), + ok = emqx_trace_handler:uninstall(ip_address, <<"CLI-IP-1">>), + ok = emqx_trace_handler:uninstall(ip_address, <<"CLI-IP-2">>), {error, _Reason} = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.2">>), emqtt:disconnect(T), ?assertEqual([], emqx_trace_handler:running()). @@ -215,7 +216,12 @@ filesync(Name, Type) -> %% sometime the handler process is not started yet. filesync(_Name, _Type, 0) -> ok; -filesync(Name, Type, Retry) -> +filesync(Name0, Type, Retry) -> + Name = + case is_binary(Name0) of + true -> Name0; + false -> list_to_binary(Name0) + end, try Handler = binary_to_atom(<<"trace_", (atom_to_binary(Type))/binary, "_", Name/binary>>), diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 5669e5653..d6902d123 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -142,11 +142,11 @@ fields(trace) -> #{desc => """Filter type""", nullable => false, example => <<"clientid">>})}, - {topic, hoconsc:mk(emqx_schema:unicode_binary(), + {topic, hoconsc:mk(binary(), #{desc => """support mqtt wildcard topic.""", nullable => true, example => <<"/dev/#">>})}, - {clientid, hoconsc:mk(emqx_schema:unicode_binary(), + {clientid, hoconsc:mk(binary(), #{desc => """mqtt clientid.""", nullable => true, example => <<"dev-001">>})}, diff --git a/apps/emqx_modules/etc/emqx_modules.conf.rendered b/apps/emqx_modules/etc/emqx_modules.conf.rendered new file mode 100644 index 000000000..be34479e8 --- /dev/null +++ b/apps/emqx_modules/etc/emqx_modules.conf.rendered @@ -0,0 +1,50 @@ + +delayed { + enable = true + ## 0 is no limit + max_delayed_messages = 0 +} + +observer_cli { + enable = true +} + +telemetry { + enable = true +} + +event_message { + "$event/client_connected" = true + "$event/client_disconnected" = true + # "$event/client_subscribed": false + # "$event/client_unsubscribed": false + # "$event/message_delivered": false + # "$event/message_acked": false + # "$event/message_dropped": false +} + +topic_metrics: [ + #{topic: "test/1"} +] + +rewrite: [ + # { + # action = publish + # source_topic = "x/#" + # re = "^x/y/(.+)$" + # dest_topic = "z/y/$1" + # }, + # { + # action = subscribe + # source_topic = "x1/#" + # re = "^x1/y/(.+)$" + # dest_topic = "z1/y/$1" + # }, + # { + # action = all + # source_topic = "x2/#" + # re = "^x2/y/(.+)$" + # dest_topic = "z2/y/$1" + # } +] + From 6280c8c9f093f52e55ca84080f2fb0d792f978eb Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 29 Dec 2021 14:02:45 +0800 Subject: [PATCH 31/99] chore: fix dialyzer warnings --- apps/emqx_gateway/src/emqx_gateway_utils.erl | 2 +- apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 3fd3045cb..95720ff13 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -144,7 +144,7 @@ start_listeners([L | Ls], GwName, Ctx, ModCfg, Acc) -> Ctx :: emqx_gateway_ctx:context(), Listener :: tuple(), ModCfg :: map()) - -> {ok, pid()} + -> {ok, {ListenerId :: atom(), esockd:listen_on(), pid()}} | {error, term()}. start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}, ModCfg) -> diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl index edf035240..47ed722b1 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl @@ -62,7 +62,7 @@ on_gateway_load(_Gateway = #{ name := GwName, {ok, ListenerPids} -> {ok, ListenerPids, #{ctx => Ctx, registry => RegPid}}; {error, {Reason, Listener}} -> - emqx_lwm2m_xml_object_db:stop(), + _ = emqx_lwm2m_xml_object_db:stop(), throw({badconf, #{ key => listeners , vallue => Listener , reason => Reason From 121d90699244dd33b5731f64d0f4d1224f541766 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 29 Dec 2021 12:49:56 +0800 Subject: [PATCH 32/99] chore(log): add SLOG/3 to add meta info --- apps/emqx/include/logger.hrl | 22 +++++++++++-------- apps/emqx/src/emqx_authentication_config.erl | 4 ++-- apps/emqx/src/emqx_broker.erl | 13 ++++++----- apps/emqx/src/emqx_channel.erl | 12 +++++----- apps/emqx/src/emqx_cm.erl | 9 +++++--- apps/emqx/src/emqx_config.erl | 4 ++-- apps/emqx/src/emqx_connection.erl | 4 ++-- apps/emqx/src/emqx_flapping.erl | 4 ++-- apps/emqx/src/emqx_session.erl | 12 ++++++---- apps/emqx/src/emqx_session_router.erl | 2 +- apps/emqx/src/emqx_trace/emqx_trace.erl | 6 ++++- apps/emqx/src/emqx_ws_connection.erl | 4 ++-- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 4 +--- apps/emqx_bridge/src/emqx_bridge.erl | 4 ++-- apps/emqx_conf/src/emqx_cluster_rpc.erl | 6 ++--- apps/emqx_conf/src/emqx_conf.erl | 2 +- .../src/emqx_connector_http.erl | 8 +++---- .../src/emqx_connector_ldap.erl | 4 ++-- .../src/emqx_connector_mongo.erl | 2 +- .../src/emqx_connector_mysql.erl | 4 ++-- .../src/emqx_connector_pgsql.erl | 2 +- .../src/emqx_connector_redis.erl | 4 ++-- .../src/mqtt/emqx_connector_mqtt_mod.erl | 8 +++---- .../src/mqtt/emqx_connector_mqtt_worker.erl | 12 +++++----- 24 files changed, 86 insertions(+), 70 deletions(-) diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index 42d598ef9..ddd4349bb 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -59,15 +59,19 @@ %% structured logging -define(SLOG(Level, Data), - %% check 'allow' here, only evaluate Data when necessary - case logger:allow(Level, ?MODULE) of - true -> - logger:log(Level, (Data), #{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} - , line => ?LINE - }); - false -> - ok - end). + ?SLOG(Level, Data, #{})). + +%% structured logging, meta is for handler's filter. +-define(SLOG(Level, Data, Meta), +%% check 'allow' here, only evaluate Data when necessary + case logger:allow(Level, ?MODULE) of + true -> + logger:log(Level, (Data), Meta#{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} + , line => ?LINE + }); + false -> + ok + end). -define(TRACE(Event, Msg, Meta), emqx_trace:log(Event, Msg, Meta)). diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index 795dd060e..9767a2265 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -187,7 +187,7 @@ convert_certs(CertsDir, Config) -> {ok, SSL} -> new_ssl_config(Config, SSL); {error, Reason} -> - ?SLOG(error, Reason#{msg => bad_ssl_config}), + ?SLOG(error, Reason#{msg => "bad_ssl_config"}), throw({bad_ssl_config, Reason}) end. @@ -199,7 +199,7 @@ convert_certs(CertsDir, NewConfig, OldConfig) -> ok = emqx_tls_lib:delete_ssl_files(CertsDir, NewSSL1, OldSSL), new_ssl_config(NewConfig, NewSSL1); {error, Reason} -> - ?SLOG(error, Reason#{msg => bad_ssl_config}), + ?SLOG(error, Reason#{msg => "bad_ssl_config"}), throw({bad_ssl_config, Reason}) end. diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index dec753fc2..4085b6130 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -204,8 +204,9 @@ publish(Msg) when is_record(Msg, message) -> _ = emqx_trace:publish(Msg), emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'), case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of - #message{headers = #{allow_publish := false}} -> - ?TRACE("MQTT", "msg_publish_not_allowed", #{message => emqx_message:to_log_map(Msg)}), + #message{headers = #{allow_publish := false}, topic = Topic} -> + Message = emqx_message:to_log_map(Msg), + ?TRACE("MQTT", "msg_publish_not_allowed", #{message => Message, topic => Topic}), []; Msg1 = #message{topic = Topic} -> emqx_persistent_session:persist_message(Msg1), @@ -225,7 +226,9 @@ safe_publish(Msg) when is_record(Msg, message) -> reason => Reason, payload => emqx_message:to_log_map(Msg), stacktrace => Stk - }), + }, + #{topic => Msg#message.topic} + ), [] end. @@ -279,7 +282,7 @@ forward(Node, To, Delivery, async) -> msg => "async_forward_msg_to_node_failed", node => Node, reason => Reason - }), + }, #{topic => To}), {error, badrpc} end; @@ -290,7 +293,7 @@ forward(Node, To, Delivery, sync) -> msg => "sync_forward_msg_to_node_failed", node => Node, reason => Reason - }), + }, #{topic => To}), {error, badrpc}; Result -> emqx_metrics:inc('messages.forward'), Result diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 0f83b04ff..d0cb07a68 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -552,7 +552,7 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) -> msg => "cannot_publish_to_topic", topic => Topic, reason => emqx_reason_codes:name(Rc) - }), + }, #{topic => Topic}), case emqx:get_config([authorization, deny_action], ignore) of ignore -> case QoS of @@ -570,7 +570,7 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) -> msg => "cannot_publish_to_topic", topic => Topic, reason => emqx_reason_codes:name(Rc) - }), + }, #{topic => Topic}), case QoS of ?QOS_0 -> ok = emqx_metrics:inc('packets.publish.dropped'), @@ -585,7 +585,7 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) -> msg => "cannot_publish_to_topic", topic => Topic, reason => emqx_reason_codes:name(Rc) - }), + }, #{topic => Topic}), handle_out(disconnect, Rc, NChannel) end. @@ -635,7 +635,7 @@ do_publish(PacketId, Msg = #message{qos = ?QOS_2}, msg => "dropped_qos2_packet", reason => emqx_reason_codes:name(RC), packet_id => PacketId - }), + }, #{topic => Msg#message.topic}), ok = emqx_metrics:inc('packets.publish.dropped'), handle_out(pubrec, {PacketId, RC}, Channel) end. @@ -687,7 +687,7 @@ process_subscribe([Topic = {TopicFilter, SubOpts} | More], SubProps, Channel, Ac ?SLOG(warning, #{ msg => "cannot_subscribe_topic_filter", reason => emqx_reason_codes:name(ReasonCode) - }), + }, #{topic => TopicFilter}), process_subscribe(More, SubProps, Channel, [{Topic, ReasonCode} | Acc]) end. @@ -703,7 +703,7 @@ do_subscribe(TopicFilter, SubOpts = #{qos := QoS}, Channel = ?SLOG(warning, #{ msg => "cannot_subscribe_topic_filter", reason => emqx_reason_codes:text(RC) - }), + }, #{topic => NTopicFilter}), {RC, Channel} end. diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 162cff2e0..c44cfe15e 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -448,20 +448,23 @@ kick_session(Action, ClientId, ChanPid) -> , action => Action , error => Error , reason => Reason - }) + }, + #{clientid => unicode:characters_to_list(ClientId, utf8)}) end. kick_session(ClientId) -> case lookup_channels(ClientId) of [] -> ?SLOG(warning, #{msg => "kicked_an_unknown_session", - clientid => ClientId}), + clientid => ClientId}, + #{clientid => unicode:characters_to_list(ClientId, utf8)}), ok; ChanPids -> case length(ChanPids) > 1 of true -> ?SLOG(warning, #{msg => "more_than_one_channel_found", - chan_pids => ChanPids}); + chan_pids => ChanPids}, + #{clientid => unicode:characters_to_list(ClientId, utf8)}); false -> ok end, lists:foreach(fun(Pid) -> kick_session(ClientId, Pid) end, ChanPids) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index f3e6e1366..35a6c048e 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -262,7 +262,7 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) -> {ok, RawRichConf} -> init_load(SchemaMod, RawRichConf); {error, Reason} -> - ?SLOG(error, #{msg => failed_to_load_hocon_conf, + ?SLOG(error, #{msg => "failed_to_load_hocon_conf", reason => Reason, include_dirs => IncDir }), @@ -396,7 +396,7 @@ save_to_override_conf(RawConf, Opts) -> case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of ok -> ok; {error, Reason} -> - ?SLOG(error, #{msg => failed_to_write_override_file, + ?SLOG(error, #{msg => "failed_to_write_override_file", filename => FileName, reason => Reason}), {error, Reason} diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 37e15f522..e1dab3260 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -872,7 +872,7 @@ check_limiter(Needs, {ok, Limiter2} -> WhenOk(Data, Msgs, State#state{limiter = Limiter2}); {pause, Time, Limiter2} -> - ?SLOG(warning, #{msg => "pause time dueto rate limit", + ?SLOG(warning, #{msg => "pause_time_dueto_rate_limit", needs => Needs, time_in_ms => Time}), @@ -912,7 +912,7 @@ retry_limiter(#state{limiter = Limiter} = State) -> , limiter_timer = undefined }); {pause, Time, Limiter2} -> - ?SLOG(warning, #{msg => "pause time dueto rate limit", + ?SLOG(warning, #{msg => "pause_time_dueto_rate_limit", types => Types, time_in_ms => Time}), diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index 600144adc..cb3da361a 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -122,7 +122,7 @@ handle_cast({detected, #flapping{clientid = ClientId, peer_host => fmt_host(PeerHost), detect_cnt => DetectCnt, wind_time_in_ms => WindTime - }), + }, #{clientid => unicode:characters_to_list(ClientId, utf8)}), Now = erlang:system_time(second), Banned = #banned{who = {clientid, ClientId}, by = <<"flapping detector">>, @@ -138,7 +138,7 @@ handle_cast({detected, #flapping{clientid = ClientId, peer_host => fmt_host(PeerHost), detect_cnt => DetectCnt, interval => Interval - }) + }, #{clientid => unicode:characters_to_list(ClientId, utf8)}) end, {noreply, State}; diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index bf79085af..1695ed6ce 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -535,16 +535,20 @@ enqueue(Msg, Session = #session{mqueue = Q}) when is_record(Msg, message) -> (Dropped =/= undefined) andalso log_dropped(Dropped, Session), Session#session{mqueue = NewQ}. -log_dropped(Msg = #message{qos = QoS}, #session{mqueue = Q}) -> - case (QoS == ?QOS_0) andalso (not emqx_mqueue:info(store_qos0, Q)) of +log_dropped(Msg = #message{qos = QoS, topic = Topic}, #session{mqueue = Q}) -> + Payload = emqx_message:to_log_map(Msg), + #{store_qos0 := StoreQos0} = QueueInfo = emqx_mqueue:info(Q), + case (QoS == ?QOS_0) andalso (not StoreQos0) of true -> ok = emqx_metrics:inc('delivery.dropped.qos0_msg'), ?SLOG(warning, #{msg => "dropped_qos0_msg", - payload => emqx_message:to_log_map(Msg)}); + queue => QueueInfo, + payload => Payload}, #{topic => Topic}); false -> ok = emqx_metrics:inc('delivery.dropped.queue_full'), ?SLOG(warning, #{msg => "dropped_msg_due_to_mqueue_is_full", - payload => emqx_message:to_log_map(Msg)}) + queue => QueueInfo, + payload => Payload}, #{topic => Topic}) end. enrich_fun(Session = #session{subscriptions = Subs}) -> diff --git a/apps/emqx/src/emqx_session_router.erl b/apps/emqx/src/emqx_session_router.erl index aaaedcb12..3d3722c32 100644 --- a/apps/emqx/src/emqx_session_router.erl +++ b/apps/emqx/src/emqx_session_router.erl @@ -260,7 +260,7 @@ code_change(_OldVsn, State, _Extra) -> init_resume_worker(RemotePid, SessionID, #{ pmon := Pmon } = State) -> case emqx_session_router_worker_sup:start_worker(SessionID, RemotePid) of {error, What} -> - ?SLOG(error, #{msg => "Could not start resume worker", reason => What}), + ?SLOG(error, #{msg => "failed_to_start_resume_worker", reason => What}), error; {ok, Pid} -> Pmon1 = emqx_pmon:monitor(Pid, Pmon), diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 67542a2bf..3869d300d 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -98,7 +98,11 @@ log(Event, Msg, Meta0) -> case persistent_term:get(?TRACE_FILTER, undefined) of undefined -> ok; List -> - Meta = maps:merge(logger:get_process_metadata(), Meta0), + Meta = + case logger:get_process_metadata() of + undefined -> Meta0; + ProcMeta -> maps:merge(ProcMeta, Meta0) + end, Log = #{level => trace, event => Event, meta => Meta, msg => Msg}, log_filter(List, Log) end. diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index f70d7ac6c..e2bdf6c72 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -549,7 +549,7 @@ check_limiter(Needs, {ok, Limiter2} -> WhenOk(Data, Msgs, State#state{limiter = Limiter2}); {pause, Time, Limiter2} -> - ?SLOG(warning, #{msg => "pause time dueto rate limit", + ?SLOG(warning, #{msg => "pause_time_due_to_rate_limit", needs => Needs, time_in_ms => Time}), @@ -585,7 +585,7 @@ retry_limiter(#state{limiter = Limiter} = State) -> , limiter_timer = undefined }); {pause, Time, Limiter2} -> - ?SLOG(warning, #{msg => "pause time dueto rate limit", + ?SLOG(warning, #{msg => "pause_time_due_to_rate_limit", types => Types, time_in_ms => Time}), diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 885811fec..820eeb859 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -43,7 +43,6 @@ groups() -> []. init_per_testcase(_, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authn_test_lib:delete_authenticators( [?CONF_NS_ATOM], ?GLOBAL), @@ -56,9 +55,8 @@ init_per_testcase(_, Config) -> Config. init_per_suite(Config) -> - _ = application:load(emqx_conf), ok = emqx_common_test_helpers:start_apps( - [emqx_authn, emqx_dashboard], + [emqx_conf, emqx_authn, emqx_dashboard], fun set_special_configs/1), ?AUTHN:delete_chain(?GLOBAL), diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index d4fc3df2d..a6681d3f1 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -214,7 +214,7 @@ update(Type, Name, {OldConf, Conf}) -> case recreate(Type, Name, Conf) of {ok, _} -> maybe_disable_bridge(Type, Name, Conf); {error, not_found} -> - ?SLOG(warning, #{ msg => "updating a non-exist bridge, create a new one" + ?SLOG(warning, #{ msg => "updating_a_non-exist_bridge_need_create_a_new_one" , type => Type, name => Name, config => Conf}), create(Type, Name, Conf); {error, Reason} -> {update_bridge_failed, Reason} @@ -242,7 +242,7 @@ create_dry_run(Type, Conf) -> end. remove(Type, Name, _Conf) -> - ?SLOG(info, #{msg => "remove bridge", type => Type, name => Name}), + ?SLOG(info, #{msg => "remove_bridge", type => Type, name => Name}), case emqx_resource:remove_local(resource_id(Type, Name)) of ok -> ok; {error, not_found} -> ok; diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 7ebe7645b..514b9156a 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -236,7 +236,7 @@ catch_up(#{node := Node, retry_interval := RetryMs} = State, SkipResult) -> false -> RetryMs end; {aborted, Reason} -> - ?SLOG(error, #{msg => "read_next_mfa transaction failed", error => Reason}), + ?SLOG(error, #{msg => "read_next_mfa_transaction_failed", error => Reason}), RetryMs end. @@ -248,7 +248,7 @@ read_next_mfa(Node) -> TnxId = max(LatestId - 1, 0), commit(Node, TnxId), ?SLOG(notice, #{ - msg => "New node first catch up and start commit.", + msg => "new_node_first_catch_up_and_start_commit.", node => Node, tnx_id => TnxId}), TnxId; [#cluster_rpc_commit{tnx_id = LastAppliedID}] -> LastAppliedID + 1 @@ -277,7 +277,7 @@ do_catch_up(ToTnxId, Node) -> io_lib:format("~p catch up failed by LastAppliedId(~p) > ToTnxId(~p)", [Node, LastAppliedId, ToTnxId])), ?SLOG(error, #{ - msg => "catch up failed!", + msg => "catch_up_failed!", last_applied_id => LastAppliedId, to_tnx_id => ToTnxId }), diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index dec07f35c..b8a8c211d 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -144,7 +144,7 @@ multicall(M, F, Args) -> {retry, TnxId, Res, Nodes} -> %% The init MFA return ok, but other nodes failed. %% We return ok and alert an alarm. - ?SLOG(error, #{msg => "failed to update config in cluster", nodes => Nodes, + ?SLOG(error, #{msg => "failed_to_update_config_in_cluster", nodes => Nodes, tnx_id => TnxId, mfa => {M, F, Args}}), Res; {error, Error} -> %% all MFA return not ok or {ok, term()}. diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 7d7771503..f597a5c1d 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -143,7 +143,7 @@ on_start(InstId, #{base_url := #{scheme := Scheme, retry_interval := RetryInterval, pool_type := PoolType, pool_size := PoolSize} = Config) -> - ?SLOG(info, #{msg => "starting http connector", + ?SLOG(info, #{msg => "starting_http_connector", connector => InstId, config => Config}), {Transport, TransportOpts} = case Scheme of http -> @@ -181,13 +181,13 @@ on_start(InstId, #{base_url := #{scheme := Scheme, end. on_stop(InstId, #{pool_name := PoolName}) -> - ?SLOG(info, #{msg => "stopping http connector", + ?SLOG(info, #{msg => "stopping_http_connector", connector => InstId}), ehttpc_sup:stop_pool(PoolName). on_query(InstId, {send_message, Msg}, AfterQuery, State) -> case maps:get(request, State, undefined) of - undefined -> ?SLOG(error, #{msg => "request not found", connector => InstId}); + undefined -> ?SLOG(error, #{msg => "request_not_found", connector => InstId}); Request -> #{method := Method, path := Path, body := Body, headers := Headers, request_timeout := Timeout} = process_request(Request, Msg), @@ -207,7 +207,7 @@ on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery, _ -> {PoolName, KeyOrNum} end, Method, NRequest, Timeout) of {error, Reason} -> - ?SLOG(error, #{msg => "http connector do reqeust failed", + ?SLOG(error, #{msg => "http_connector_do_reqeust_failed", request => NRequest, reason => Reason, connector => InstId}), emqx_resource:query_failed(AfterQuery); diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index 97e963f18..c188837ba 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -55,7 +55,7 @@ on_start(InstId, #{servers := Servers0, pool_size := PoolSize, auto_reconnect := AutoReconn, ssl := SSL} = Config) -> - ?SLOG(info, #{msg => "starting ldap connector", + ?SLOG(info, #{msg => "starting_ldap_connector", connector => InstId, config => Config}), Servers = [begin proplists:get_value(host, S) end || S <- Servers0], SslOpts = case maps:get(enable, SSL) of @@ -81,7 +81,7 @@ on_start(InstId, #{servers := Servers0, {ok, #{poolname => PoolName}}. on_stop(InstId, #{poolname := PoolName}) -> - ?SLOG(info, #{msg => "stopping ldap connector", + ?SLOG(info, #{msg => "stopping_ldap_connector", connector => InstId}), emqx_plugin_libs_pool:stop_pool(PoolName). diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index eacb3ec2d..5321ea459 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -128,7 +128,7 @@ on_start(InstId, Config = #{mongo_type := Type, {ok, #{poolname => PoolName, type => Type}}. on_stop(InstId, #{poolname := PoolName}) -> - ?SLOG(info, #{msg => "stopping mongodb connector", + ?SLOG(info, #{msg => "stopping_mongodb_connector", connector => InstId}), emqx_plugin_libs_pool:stop_pool(PoolName). diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index ae8239936..265a6a01e 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -56,7 +56,7 @@ on_start(InstId, #{server := {Host, Port}, auto_reconnect := AutoReconn, pool_size := PoolSize, ssl := SSL } = Config) -> - ?SLOG(info, #{msg => "starting mysql connector", + ?SLOG(info, #{msg => "starting_mysql_connector", connector => InstId, config => Config}), SslOpts = case maps:get(enable, SSL) of true -> @@ -76,7 +76,7 @@ on_start(InstId, #{server := {Host, Port}, {ok, #{poolname => PoolName}}. on_stop(InstId, #{poolname := PoolName}) -> - ?SLOG(info, #{msg => "stopping mysql connector", + ?SLOG(info, #{msg => "stopping_mysql_connector", connector => InstId}), emqx_plugin_libs_pool:stop_pool(PoolName). diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 9b6f559b4..36bff3386 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -56,7 +56,7 @@ on_start(InstId, #{server := {Host, Port}, auto_reconnect := AutoReconn, pool_size := PoolSize, ssl := SSL } = Config) -> - ?SLOG(info, #{msg => "starting postgresql connector", + ?SLOG(info, #{msg => "starting_postgresql_connector", connector => InstId, config => Config}), SslOpts = case maps:get(enable, SSL) of true -> diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 94f4eca3e..48001ca25 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -87,7 +87,7 @@ on_start(InstId, #{redis_type := Type, pool_size := PoolSize, auto_reconnect := AutoReconn, ssl := SSL } = Config) -> - ?SLOG(info, #{msg => "starting redis connector", + ?SLOG(info, #{msg => "starting_redis_connector", connector => InstId, config => Config}), Servers = case Type of single -> [{servers, [maps:get(server, Config)]}]; @@ -120,7 +120,7 @@ on_start(InstId, #{redis_type := Type, {ok, #{poolname => PoolName, type => Type}}. on_stop(InstId, #{poolname := PoolName}) -> - ?SLOG(info, #{msg => "stopping redis connector", + ?SLOG(info, #{msg => "stopping_redis_connector", connector => InstId}), emqx_plugin_libs_pool:stop_pool(PoolName). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl index 7d5bb1283..3ab410391 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl @@ -158,15 +158,15 @@ handle_puback(#{packet_id := PktId, reason_code := RC}, Parent) RC =:= ?RC_NO_MATCHING_SUBSCRIBERS -> Parent ! {batch_ack, PktId}, ok; handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) -> - ?SLOG(warning, #{msg => "publish to remote node falied", + ?SLOG(warning, #{msg => "publish_to_remote_node_falied", packet_id => PktId, reason_code => RC}). handle_publish(Msg, undefined) -> - ?SLOG(error, #{msg => "cannot publish to local broker as" - " 'ingress' is not configured", + ?SLOG(error, #{msg => "cannot_publish_to_local_broker_as" + "_'ingress'_is_not_configured", message => Msg}); handle_publish(Msg, Vars) -> - ?SLOG(debug, #{msg => "publish to local broker", + ?SLOG(debug, #{msg => "publish_to_local_broker", message => Msg, vars => Vars}), emqx_metrics:inc('bridge.mqtt.message_received_from_remote', 1), case Vars of diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 5f6f4b69f..e0d5a2d77 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -188,7 +188,7 @@ callback_mode() -> [state_functions]. %% @doc Config should be a map(). init(#{name := Name} = ConnectOpts) -> - ?SLOG(debug, #{msg => "starting bridge worker", + ?SLOG(debug, #{msg => "starting_bridge_worker", name => Name}), erlang:process_flag(trap_exit, true), Queue = open_replayq(Name, maps:get(replayq, ConnectOpts, #{})), @@ -335,7 +335,7 @@ common(_StateName, cast, {send_to_remote, Msg}, #{replayq := Q} = State) -> NewQ = replayq:append(Q, [Msg]), {keep_state, State#{replayq => NewQ}, {next_event, internal, maybe_send}}; common(StateName, Type, Content, #{name := Name} = State) -> - ?SLOG(notice, #{msg => "Bridge discarded event", + ?SLOG(notice, #{msg => "bridge_discarded_event", name => Name, type => Type, state_name => StateName, content => Content}), {keep_state, State}. @@ -349,7 +349,7 @@ do_connect(#{connect_opts := ConnectOpts, {ok, State#{connection => Conn}}; {error, Reason} -> ConnectOpts1 = obfuscate(ConnectOpts), - ?SLOG(error, #{msg => "Failed to connect", + ?SLOG(error, #{msg => "failed_to_connect", config => ConnectOpts1, reason => Reason}), {error, Reason, State} end. @@ -386,8 +386,8 @@ pop_and_send_loop(#{replayq := Q} = State, N) -> end. do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Msg) -> - ?SLOG(error, #{msg => "cannot forward messages to remote broker" - " as 'egress' is not configured", + ?SLOG(error, #{msg => "cannot_forward_messages_to_remote_broker" + "_as_'egress'_is_not_configured", messages => Msg}); do_send(#{inflight := Inflight, connection := Connection, @@ -398,7 +398,7 @@ do_send(#{inflight := Inflight, emqx_metrics:inc('bridge.mqtt.message_sent_to_remote'), emqx_connector_mqtt_msg:to_remote_msg(Message, Vars) end, - ?SLOG(debug, #{msg => "publish to remote broker", + ?SLOG(debug, #{msg => "publish_to_remote_broker", message => Msg, vars => Vars}), case emqx_connector_mqtt_mod:send(Connection, [ExportMsg(Msg)]) of {ok, Refs} -> From 04313dc044f7d9ba4d32bac2ba4ca1f9973154c2 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 29 Dec 2021 19:03:54 +0800 Subject: [PATCH 33/99] fix(trace): download trace api not working --- .../src/emqx_dashboard_swagger.erl | 3 ++ .../src/emqx_mgmt_api_trace.erl | 26 +++++++--- .../etc/emqx_modules.conf.rendered | 50 ------------------- rebar.config | 2 +- 4 files changed, 24 insertions(+), 57 deletions(-) delete mode 100644 apps/emqx_modules/etc/emqx_modules.conf.rendered diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 9a54be9c5..bfc6ae951 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -312,6 +312,9 @@ responses(Responses, Module) -> response(Status, Bin, {Acc, RefsAcc, Module}) when is_binary(Bin) -> {Acc#{integer_to_binary(Status) => #{description => Bin}}, RefsAcc, Module}; +%% Support swagger raw object(file download). +response(Status, #{content := _} = Content, {Acc, RefsAcc, Module}) -> + {Acc#{integer_to_binary(Status) => Content}, RefsAcc, Module}; response(Status, ?REF(StructName), {Acc, RefsAcc, Module}) -> response(Status, ?R_REF(Module, StructName), {Acc, RefsAcc, Module}); response(Status, ?R_REF(_Mod, _Name) = RRef, {Acc, RefsAcc, Module}) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index d6902d123..15277fa8e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -107,9 +107,14 @@ schema("/trace/:name/download") -> get => #{ description => "Download trace log by name", parameters => [hoconsc:ref(name)], - %% todo zip file octet-stream responses => #{ - 200 => <<"TODO octet-stream">> + 200 => + #{description => "A trace zip file", + content => #{ + 'application/octet-stream' => + #{schema => #{type => "string", format => "binary"}} + } + } } } }; @@ -126,7 +131,11 @@ schema("/trace/:name/log") -> ], %% todo response data responses => #{ - 200 => <<"TODO">> + 200 => + [ + {items, hoconsc:mk(binary(), #{example => "BinBinBin"})} + | fields(bytes) ++ fields(position) + ] } } }. @@ -209,6 +218,7 @@ fields(position) -> default => 0 })}]. + -define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_]*$"). validate_name(Name) -> @@ -296,7 +306,12 @@ download_trace_log(get, #{bindings := #{name := Name}}) -> ZipFileName = ZipDir ++ binary_to_list(Name) ++ ".zip", {ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]), emqx_trace:delete_files_after_send(ZipFileName, Zips), - {200, ZipFile}; + Headers = #{ + <<"content-type">> => <<"application/octet-stream">>, + <<"content-disposition">> => + iolist_to_binary("attachment; filename=" ++ filename:basename(ZipFile)) + }, + {200, Headers, {file, ZipFile}}; {error, not_found} -> ?NOT_FOUND(Name) end. @@ -324,11 +339,10 @@ cluster_call(Mod, Fun, Args, Timeout) -> BadNodes =/= [] andalso ?LOG(error, "rpc call failed on ~p ~p", [BadNodes, {Mod, Fun, Args}]), GoodRes. -stream_log_file(get, #{bindings := #{name := Name}, query_string := Query} = T) -> +stream_log_file(get, #{bindings := #{name := Name}, query_string := Query}) -> Node0 = maps:get(<<"node">>, Query, atom_to_binary(node())), Position = maps:get(<<"position">>, Query, 0), Bytes = maps:get(<<"bytes">>, Query, 1000), - logger:error("~p", [T]), case to_node(Node0) of {ok, Node} -> case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of diff --git a/apps/emqx_modules/etc/emqx_modules.conf.rendered b/apps/emqx_modules/etc/emqx_modules.conf.rendered deleted file mode 100644 index be34479e8..000000000 --- a/apps/emqx_modules/etc/emqx_modules.conf.rendered +++ /dev/null @@ -1,50 +0,0 @@ - -delayed { - enable = true - ## 0 is no limit - max_delayed_messages = 0 -} - -observer_cli { - enable = true -} - -telemetry { - enable = true -} - -event_message { - "$event/client_connected" = true - "$event/client_disconnected" = true - # "$event/client_subscribed": false - # "$event/client_unsubscribed": false - # "$event/message_delivered": false - # "$event/message_acked": false - # "$event/message_dropped": false -} - -topic_metrics: [ - #{topic: "test/1"} -] - -rewrite: [ - # { - # action = publish - # source_topic = "x/#" - # re = "^x/y/(.+)$" - # dest_topic = "z/y/$1" - # }, - # { - # action = subscribe - # source_topic = "x1/#" - # re = "^x1/y/(.+)$" - # dest_topic = "z1/y/$1" - # }, - # { - # action = all - # source_topic = "x2/#" - # re = "^x2/y/(.+)$" - # dest_topic = "z2/y/$1" - # } -] - diff --git a/rebar.config b/rebar.config index 1894d817d..14fe8eb65 100644 --- a/rebar.config +++ b/rebar.config @@ -55,7 +55,7 @@ , {mria, {git, "https://github.com/emqx/mria", {tag, "0.1.5"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.7"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.9"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} , {replayq, "0.3.3"} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} From 34dfc327f36ad8ae3ba9635d94ed8bcce84e896b Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Wed, 29 Dec 2021 19:32:05 +0800 Subject: [PATCH 34/99] fix(api): auth failed with basic type check --- apps/emqx_dashboard/src/emqx_dashboard.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 0c7c03f63..a85db870c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -151,9 +151,9 @@ authorize(Req) -> ok -> ok; {error, token_timeout} -> - return_unauthorized(<<"TOKEN_TIME_OUT">>, <<"POST '/login', get new token">>); + {401, <<"TOKEN_TIME_OUT">>, <<"POST '/login', get new token">>}; {error, not_found} -> - return_unauthorized(<<"BAD_TOKEN">>, <<"POST '/login'">>) + {401, <<"BAD_TOKEN">>, <<"POST '/login', get new token">>} end; _ -> return_unauthorized(<<"AUTHORIZATION_HEADER_ERROR">>, From b8bb5ff738704ace0f5c6266e1f74b819c10fe53 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 29 Dec 2021 22:33:34 +0800 Subject: [PATCH 35/99] fix(trace): delete duplicate topic from msg --- apps/emqx/etc/emqx.conf.rendered | 1671 ----------------- apps/emqx/include/logger.hrl | 6 +- apps/emqx/src/emqx_channel.erl | 2 - apps/emqx/src/emqx_cm.erl | 3 +- apps/emqx/src/emqx_flapping.erl | 2 - .../src/emqx_trace/emqx_trace_handler.erl | 15 - 6 files changed, 4 insertions(+), 1695 deletions(-) delete mode 100644 apps/emqx/etc/emqx.conf.rendered diff --git a/apps/emqx/etc/emqx.conf.rendered b/apps/emqx/etc/emqx.conf.rendered deleted file mode 100644 index afa640621..000000000 --- a/apps/emqx/etc/emqx.conf.rendered +++ /dev/null @@ -1,1671 +0,0 @@ -##================================================================== -## Listeners -##================================================================== -## MQTT/TCP - TCP Listeners for MQTT Protocol -## syntax: listeners.tcp. -## example: listeners.tcp.my_tcp_listener -listeners.tcp.default { - ## The IP address and port that the listener will bind. - ## - ## @doc listeners.tcp..bind - ## ValueType: IPAddress | Port | IPAddrPort - ## Required: true - ## Examples: 1883, 127.0.0.1:1883, ::1:1883 - bind = "0.0.0.0:1883" - - ## The configuration zone this listener is using. - ## If not set, the global configs are used for this listener. - ## - ## See `zones.` for more details. - ## - ## @doc listeners.tcp..zone - ## ValueType: String - ## Required: false - #zone = default - - ## The size of the acceptor pool for this listener. - ## - ## @doc listeners.tcp..acceptors - ## ValueType: Number - ## Default: 16 - acceptors = 16 - - ## Maximum number of concurrent connections. - ## - ## @doc listeners.tcp..max_connections - ## ValueType: Number | infinity - ## Default: infinity - max_connections = 1024000 - - ## The access control rules for this listener. - ## - ## See: https://github.com/emqtt/esockd#allowdeny - ## - ## @doc listeners.tcp..access_rules - ## ValueType: Array - ## Default: [] - ## Examples: - ## access_rules: [ - ## "deny 192.168.0.0/24", - ## "all all" - ## ] - access_rules = [ - "allow all" - ] - - ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed - ## behind HAProxy or Nginx. - ## - ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ - ## - ## @doc listeners.tcp..proxy_protocol - ## ValueType: Boolean - ## Default: false - proxy_protocol = false - - ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection - ## if no proxy protocol packet received within the timeout. - ## - ## @doc listeners.tcp..proxy_protocol_timeout - ## ValueType: Duration - ## Default: 3s - proxy_protocol_timeout = 3s - - ## When publishing or subscribing, prefix all topics with a mountpoint string. - ## The prefixed string will be removed from the topic name when the message - ## is delivered to the subscriber. The mountpoint is a way that users can use - ## to implement isolation of message routing between different listeners. - ## - ## For example if a clientA subscribes to "t" with `listeners.tcp..mountpoint` - ## set to "some_tenant", then the client accually subscribes to the topic - ## "some_tenant/t". Similarly if another clientB (connected to the same listener - ## with the clientA) send a message to topic "t", the message is accually route - ## to all the clients subscribed "some_tenant/t", so clientA will receive the - ## message, with topic name "t". - ## - ## Set to "" to disable the feature. - ## - ## Variables in mountpoint string: - ## - ${clientid}: clientid - ## - ${username}: username - ## - ## @doc listeners.tcp..mountpoint - ## ValueType: String - ## Default: "" - mountpoint = "" - - ## TCP options - ## See ${example_common_tcp_options} for more information - tcp.backlog = 1024 - tcp.buffer = 4KB -} - -## MQTT/SSL - SSL Listeners for MQTT Protocol -## syntax: listeners.ssl. -## example: listeners.ssl.my_ssl_listener -listeners.ssl.default { - ## The IP address and port that the listener will bind. - ## - ## @doc listeners.ssl..bind - ## ValueType: IPAddress | Port | IPAddrPort - ## Required: true - ## Examples: 8883, 127.0.0.1:8883, ::1:8883 - bind = "0.0.0.0:8883" - - ## The configuration zone this listener is using. - ## If not set, the global configs are used for this listener. - ## - ## See `zones.` for more details. - ## - ## @doc listeners.ssl..zone - ## ValueType: String - ## Required: false - #zone = default - - ## The size of the acceptor pool for this listener. - ## - ## @doc listeners.ssl..acceptors - ## ValueType: Number - ## Default: 16 - acceptors = 16 - - ## Maximum number of concurrent connections. - ## - ## @doc listeners.ssl..max_connections - ## ValueType: Number | infinity - ## Default: infinity - max_connections = 512000 - - ## The access control rules for this listener. - ## - ## See: https://github.com/emqtt/esockd#allowdeny - ## - ## @doc listeners.ssl..access_rules - ## ValueType: Array - ## Default: [] - ## Examples: - ## access_rules: [ - ## "deny 192.168.0.0/24", - ## "all all" - ## ] - access_rules = [ - "allow all" - ] - - ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed - ## behind HAProxy or Nginx. - ## - ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ - ## - ## @doc listeners.ssl..proxy_protocol - ## ValueType: Boolean - ## Default: true - proxy_protocol = false - - ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection - ## if no proxy protocol packet received within the timeout. - ## - ## @doc listeners.ssl..proxy_protocol_timeout - ## ValueType: Duration - ## Default: 3s - proxy_protocol_timeout = 3s - - ## When publishing or subscribing, prefix all topics with a mountpoint string. - ## The prefixed string will be removed from the topic name when the message - ## is delivered to the subscriber. The mountpoint is a way that users can use - ## to implement isolation of message routing between different listeners. - ## - ## For example if a clientA subscribes to "t" with `listeners.ssl..mountpoint` - ## set to "some_tenant", then the client accually subscribes to the topic - ## "some_tenant/t". Similarly if another clientB (connected to the same listener - ## with the clientA) send a message to topic "t", the message is accually route - ## to all the clients subscribed "some_tenant/t", so clientA will receive the - ## message, with topic name "t". - ## - ## Set to "" to disable the feature. - ## - ## Variables in mountpoint string: - ## - ${clientid}: clientid - ## - ${username}: username - ## - ## @doc listeners.ssl..mountpoint - ## ValueType: String - ## Default: "" - mountpoint = "" - - ## SSL options - ssl.keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem" - ssl.certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem" - ssl.cacertfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem" - - # ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] - # TLS 1.3: "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256" - # TLS 1-1.2 "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" - # PSK: "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" - # NOTE: If PSK cipher-suites are intended, tlsv1.3 should not be enabled in 'versions' config - # ssl.ciphers = "" - - ## TCP options - ## See ${example_common_tcp_options} for more information - tcp.backlog = 1024 - tcp.buffer = 4KB -} - -## MQTT/QUIC - QUIC Listeners for MQTT Protocol -## syntax: listeners.quic. -## example: listeners.quic.my_quic_listener -listeners.quic.default { - ## The IP address and port that the listener will bind. - ## - ## @doc listeners.quic..bind - ## ValueType: IPAddress | Port | IPAddrPort - ## Required: true - ## Examples: 14567, 127.0.0.1:14567, ::1:14567 - bind = "0.0.0.0:14567" - - ## The configuration zone this listener is using. - ## If not set, the global configs are used for this listener. - ## - ## See `zones.` for more details. - ## NOTE: This is a cluster-wide configuration. - ## It requires all nodes to be stopped before changing it. - ## - ## @doc listeners.quic..zone - ## ValueType: String - ## Required: false - #zone = default - - ## The size of the acceptor pool for this listener. - ## - ## @doc listeners.quic..acceptors - ## ValueType: Number - ## Default: 16 - acceptors = 16 - - ## Maximum number of concurrent connections. - ## - ## @doc listeners.quic..max_connections - ## ValueType: Number | infinity - ## Default: infinity - max_connections = 1024000 - - ## Path to the file containing the user's private PEM-encoded key. - ## - ## @doc listeners.quic..keyfile - ## ValueType: String - ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem" - keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem" - - ## Path to a file containing the user certificate. - ## - ## @doc listeners.quic..certfile - ## ValueType: String - ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem" - certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem" - - ## When publishing or subscribing, prefix all topics with a mountpoint string. - ## The prefixed string will be removed from the topic name when the message - ## is delivered to the subscriber. The mountpoint is a way that users can use - ## to implement isolation of message routing between different listeners. - ## - ## For example if a clientA subscribes to "t" with `listeners.quic..mountpoint` - ## set to "some_tenant", then the client accually subscribes to the topic - ## "some_tenant/t". Similarly if another clientB (connected to the same listener - ## with the clientA) send a message to topic "t", the message is accually route - ## to all the clients subscribed "some_tenant/t", so clientA will receive the - ## message, with topic name "t". - ## - ## Set to "" to disable the feature. - ## - ## Variables in mountpoint string: - ## - ${clientid}: clientid - ## - ${username}: username - ## - ## @doc listeners.quic..mountpoint - ## ValueType: String - ## Default: "" - mountpoint = "" -} - -## MQTT/WS - Websocket Listeners for MQTT Protocol -## syntax: listeners.ws. -## example: listeners.ws.my_ws_listener -listeners.ws.default { - ## The IP address and port that the listener will bind. - ## - ## @doc listeners.ws..bind - ## ValueType: IPAddress | Port | IPAddrPort - ## Required: true - ## Examples: 8083, 127.0.0.1:8083, ::1:8083 - bind = "0.0.0.0:8083" - - ## The configuration zone this listener is using. - ## If not set, the global configs are used for this listener. - ## - ## See `zones.` for more details. - ## - ## @doc listeners.ws..zone - ## ValueType: String - ## Required: false - #zone = default - - ## The size of the acceptor pool for this listener. - ## - ## @doc listeners.ws..acceptors - ## ValueType: Number - ## Default: 16 - acceptors = 16 - - ## Maximum number of concurrent connections. - ## - ## @doc listeners.ws..max_connections - ## ValueType: Number | infinity - ## Default: infinity - max_connections = 1024000 - - ## The access control rules for this listener. - ## - ## See: https://github.com/emqtt/esockd#allowdeny - ## - ## @doc listeners.ws..access_rules - ## ValueType: Array - ## Default: [] - ## Examples: - ## access_rules: [ - ## "deny 192.168.0.0/24", - ## "all all" - ## ] - access_rules = [ - "allow all" - ] - - ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed - ## behind HAProxy or Nginx. - ## - ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ - ## - ## @doc listeners.ws..proxy_protocol - ## ValueType: Boolean - ## Default: true - proxy_protocol = false - - ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection - ## if no proxy protocol packet received within the timeout. - ## - ## @doc listeners.ws..proxy_protocol_timeout - ## ValueType: Duration - ## Default: 3s - proxy_protocol_timeout = 3s - - ## When publishing or subscribing, prefix all topics with a mountpoint string. - ## The prefixed string will be removed from the topic name when the message - ## is delivered to the subscriber. The mountpoint is a way that users can use - ## to implement isolation of message routing between different listeners. - ## - ## For example if a clientA subscribes to "t" with `listeners.ws..mountpoint` - ## set to "some_tenant", then the client accually subscribes to the topic - ## "some_tenant/t". Similarly if another clientB (connected to the same listener - ## with the clientA) send a message to topic "t", the message is accually route - ## to all the clients subscribed "some_tenant/t", so clientA will receive the - ## message, with topic name "t". - ## - ## Set to "" to disable the feature. - ## - ## Variables in mountpoint string: - ## - ${clientid}: clientid - ## - ${username}: username - ## - ## @doc listeners.ws..mountpoint - ## ValueType: String - ## Default: "" - mountpoint = "" - - ## TCP options - ## See ${example_common_tcp_options} for more information - tcp.backlog = 1024 - tcp.buffer = 4KB - - ## Websocket options - ## See ${example_common_websocket_options} for more information - websocket.idle_timeout = 86400s -} - -## MQTT/WSS - WebSocket Secure Listeners for MQTT Protocol -## syntax: listeners.wss. -## example: listeners.wss.my_wss_listener -listeners.wss.default { - ## The IP address and port that the listener will bind. - ## - ## @doc listeners.wss..bind - ## ValueType: IPAddress | Port | IPAddrPort - ## Required: true - ## Examples: 8084, 127.0.0.1:8084, ::1:8084 - bind = "0.0.0.0:8084" - - ## The configuration zone this listener is using. - ## If not set, the global configs are used for this listener. - ## - ## See `zones.` for more details. - ## - ## @doc listeners.wss..zone - ## ValueType: String - ## Required: false - #zone = default - - ## The size of the acceptor pool for this listener. - ## - ## @doc listeners.wss..acceptors - ## ValueType: Number - ## Default: 16 - acceptors = 16 - - ## Maximum number of concurrent connections. - ## - ## @doc listeners.wss..max_connections - ## ValueType: Number | infinity - ## Default: infinity - max_connections = 512000 - - ## The access control rules for this listener. - ## - ## See: https://github.com/emqtt/esockd#allowdeny - ## - ## @doc listeners.wss..access_rules - ## ValueType: Array - ## Default: [] - ## Examples: - ## access_rules: [ - ## "deny 192.168.0.0/24", - ## "all all" - ## ] - access_rules = [ - "allow all" - ] - - ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed - ## behind HAProxy or Nginx. - ## - ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ - ## - ## @doc listeners.wss..proxy_protocol - ## ValueType: Boolean - ## Default: true - proxy_protocol = false - - ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection - ## if no proxy protocol packet received within the timeout. - ## - ## @doc listeners.wss..proxy_protocol_timeout - ## ValueType: Duration - ## Default: 3s - proxy_protocol_timeout = 3s - - ## When publishing or subscribing, prefix all topics with a mountpoint string. - ## The prefixed string will be removed from the topic name when the message - ## is delivered to the subscriber. The mountpoint is a way that users can use - ## to implement isolation of message routing between different listeners. - ## - ## For example if a clientA subscribes to "t" with `listeners.wss..mountpoint` - ## set to "some_tenant", then the client accually subscribes to the topic - ## "some_tenant/t". Similarly if another clientB (connected to the same listener - ## with the clientA) send a message to topic "t", the message is accually route - ## to all the clients subscribed "some_tenant/t", so clientA will receive the - ## message, with topic name "t". - ## - ## Set to "" to disable the feature. - ## - ## Variables in mountpoint string: - ## - ${clientid}: clientid - ## - ${username}: username - ## - ## @doc listeners.wss..mountpoint - ## ValueType: String - ## Default: "" - mountpoint = "" - - ## SSL options - ## See ${example_common_ssl_options} for more information - ssl.keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem" - ssl.certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem" - ssl.cacertfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem" - - ## TCP options - ## See ${example_common_tcp_options} for more information - tcp.backlog = 1024 - tcp.buffer = 4KB - - ## Websocket options - ## See ${example_common_websocket_options} for more information - websocket.idle_timeout = 86400s - -} - -## Enable per connection statistics. -## -## @doc stats.enable -## ValueType: Boolean -## Default: true -stats.enable = true - -authorization { - ## Behaviour after not matching a rule. - ## - ## @doc authorization.no_match - ## ValueType: allow | deny - ## Default: allow - no_match: allow - - ## The action when authorization check reject current operation - ## - ## @doc authorization.deny_action - ## ValueType: ignore | disconnect - ## Default: ignore - deny_action: ignore - - ## Whether to enable Authorization cache. - ## - ## If enabled, Authorization roles for each client will be cached in the memory - ## - ## @doc authorization.cache.enable - ## ValueType: Boolean - ## Default: true - cache.enable: true - - ## The maximum count of Authorization entries can be cached for a client. - ## - ## @doc authorization.cache.max_size - ## ValueType: Integer - ## Range: [0, 1048576] - ## Default: 32 - cache.max_size: 32 - - ## The time after which an Authorization cache entry will be deleted - ## - ## @doc authorization.cache.ttl - ## ValueType: Duration - ## Default: 1m - cache.ttl: 1m -} - -mqtt { - ## How long time the MQTT connection will be disconnected if the - ## TCP connection is established but MQTT CONNECT has not been - ## received. - ## - ## @doc mqtt.idle_timeout - ## ValueType: Duration - ## Default: 15s - idle_timeout = 15s - - ## Maximum MQTT packet size allowed. - ## - ## @doc mqtt.max_packet_size - ## ValueType: Bytes - ## Default: 1MB - max_packet_size = 1MB - - ## Maximum length of MQTT clientId allowed. - ## - ## @doc mqtt.max_clientid_len - ## ValueType: Integer - ## Range: [23, 65535] - ## Default: 65535 - max_clientid_len = 65535 - - ## Maximum topic levels allowed. - ## - ## @doc mqtt.max_topic_levels - ## ValueType: Integer - ## Range: [1, 65535] - ## Default: 128 - ## Depth so big may lead to subscribing performance issues - max_topic_levels = 128 - - ## Maximum QoS allowed. - ## - ## @doc mqtt.max_qos_allowed - ## ValueType: 0 | 1 | 2 - ## Default: 2 - max_qos_allowed = 2 - - ## Maximum Topic Alias, 0 means no topic alias supported. - ## - ## @doc mqtt.max_topic_alias - ## ValueType: Integer - ## Range: [0, 65535] - ## Default: 65535 - max_topic_alias = 65535 - - ## Whether the Server supports MQTT retained messages. - ## - ## @doc mqtt.retain_available - ## ValueType: Boolean - ## Default: true - retain_available = true - - ## Whether the Server supports MQTT Wildcard Subscriptions - ## - ## @doc mqtt.wildcard_subscription - ## ValueType: Boolean - ## Default: true - wildcard_subscription = true - - ## Whether the Server supports MQTT Shared Subscriptions. - ## - ## @doc mqtt.shared_subscription - ## ValueType: Boolean - ## Default: true - shared_subscription = true - - ## Whether to ignore loop delivery of messages.(for mqtt v3.1.1) - ## - ## @doc mqtt.ignore_loop_deliver - ## ValueType: Boolean - ## Default: false - ignore_loop_deliver = false - - ## Whether to parse the MQTT frame in strict mode - ## - ## @doc mqtt.strict_mode - ## ValueType: Boolean - ## Default: false - strict_mode = false - - ## Specify the response information returned to the client - ## - ## This feature is disabled if is set to "" - ## - ## @doc mqtt.response_information - ## ValueType: String - ## Default: "" - response_information = "" - - ## Server Keep Alive of MQTT 5.0 - ## - ## @doc mqtt.server_keepalive - ## ValueType: Number | disabled - ## Default: disabled - server_keepalive = disabled - - ## The backoff for MQTT keepalive timeout. The broker will kick a connection out - ## until 'Keepalive * backoff * 2' timeout. - ## - ## @doc mqtt.keepalive_backoff - ## ValueType: Float - ## Range: (0.5, 1] - ## Default: 0.75 - keepalive_backoff = 0.75 - - ## Maximum number of subscriptions allowed. - ## - ## @doc mqtt.max_subscriptions - ## ValueType: Integer | infinity - ## Range: [1, infinity) - ## Default: infinity - max_subscriptions = infinity - - ## Force to upgrade QoS according to subscription. - ## - ## @doc mqtt.upgrade_qos - ## ValueType: Boolean - ## Default: false - upgrade_qos = false - - ## Maximum size of the Inflight Window storing QoS1/2 messages delivered but unacked. - ## - ## @doc mqtt.max_inflight - ## ValueType: Integer - ## Range: [1, 65535] - ## Default: 32 - max_inflight = 32 - - ## Retry interval for QoS1/2 message delivering. - ## - ## @doc mqtt.retry_interval - ## ValueType: Duration - ## Default: 30s - retry_interval = 30s - - ## Maximum QoS2 packets (Client -> Broker) awaiting PUBREL. - ## - ## @doc mqtt.max_awaiting_rel - ## ValueType: Integer | infinity - ## Range: [1, infinity) - ## Default: 100 - max_awaiting_rel = 100 - - ## The QoS2 messages (Client -> Broker) will be dropped if awaiting PUBREL timeout. - ## - ## @doc mqtt.await_rel_timeout - ## ValueType: Duration - ## Default: 300s - await_rel_timeout = 300s - - ## Default session expiry interval for MQTT V3.1.1 connections. - ## - ## @doc mqtt.session_expiry_interval - ## ValueType: Duration - ## Default: 2h - session_expiry_interval = 2h - - ## Maximum queue length. Enqueued messages when persistent client disconnected, - ## or inflight window is full. - ## - ## @doc mqtt.max_mqueue_len - ## ValueType: Integer | infinity - ## Range: [0, infinity) - ## Default: 1000 - max_mqueue_len = 1000 - - ## Topic priorities. - ## - ## There's no priority table by default, hence all messages - ## are treated equal. - ## - ## Priority number [1-255] - ## - ## NOTE: comma and equal signs are not allowed for priority topic names - ## NOTE: Messages for topics not in the priority table are treated as - ## either highest or lowest priority depending on the configured - ## value for mqtt.mqueue_default_priority - ## - ## @doc mqtt.mqueue_priorities - ## ValueType: Map | disabled - ## Examples: - ## To configure "topic/1" > "topic/2": - ## mqueue_priorities: {"topic/1": 10, "topic/2": 8} - ## Default: disabled - mqueue_priorities = disabled - - ## Default to highest priority for topics not matching priority table - ## - ## @doc mqtt.mqueue_default_priority - ## ValueType: highest | lowest - ## Default: lowest - mqueue_default_priority = lowest - - ## Whether to enqueue QoS0 messages. - ## - ## @doc mqtt.mqueue_store_qos0 - ## ValueType: Boolean - ## Default: true - mqueue_store_qos0 = true - - ## Whether use username replace client id - ## - ## @doc mqtt.use_username_as_clientid - ## ValueType: Boolean - ## Default: false - use_username_as_clientid = false - - ## Use the CN, DN or CRT field from the client certificate as a username. - ## Only works for SSL connection. - ## - ## @doc mqtt.peer_cert_as_username - ## ValueType: cn | dn | crt | disabled - ## Default: disabled - peer_cert_as_username = disabled - - ## Use the CN, DN or CRT field from the client certificate as a clientid. - ## Only works for SSL connection. - ## - ## @doc mqtt.peer_cert_as_clientid - ## ValueType: cn | dn | crt | disabled - ## Default: disabled - peer_cert_as_clientid = disabled -} - -flapping_detect { - ## Enable Flapping Detection. - ## - ## This config controls the allowed maximum number of CONNECT received - ## from the same clientid in a time frame defined by `window_time`. - ## After the limit is reached, successive CONNECT requests are forbidden - ## (banned) until the end of the time period defined by `ban_time`. - ## - ## @doc flapping_detect.enable - ## ValueType: Boolean - ## Default: true - enable = false - - ## The max disconnect allowed of a MQTT Client in `window_time` - ## - ## @doc flapping_detect.max_count - ## ValueType: Integer - ## Default: 15 - max_count = 15 - - ## The time window for flapping detect - ## - ## @doc flapping_detect.window_time - ## ValueType: Duration - ## Default: 1m - window_time = 1m - - ## How long the clientid will be banned - ## - ## @doc flapping_detect.ban_time - ## ValueType: Duration - ## Default: 5m - ban_time = 5m - -} - -force_shutdown { - ## Enable force_shutdown - ## - ## @doc force_shutdown.enable - ## ValueType: Boolean - ## Default: true - enable = true - - ## Max message queue length - ## @doc force_shutdown.max_message_queue_len - ## ValueType: Integer - ## Range: (0, infinity) - ## Default: 1000 - max_message_queue_len = 1000 - - ## Total heap size - ## - ## @doc force_shutdown.max_heap_size - ## ValueType: Size - ## Default: 32MB - max_heap_size = 32MB -} - -overload_protection { - ## React on system overload or not - ## @doc overload_protection.enable - ## ValueType: Boolean - ## Default: false - enable = false - - ## Backoff delay in ms - ## @doc overload_protection.backoff_delay - ## ValueType: Integer - ## Range: (0, infinity) - ## Default: 1 - backoff_delay = 1 - - ## Backoff GC enabled - ## @doc overload_protection.backoff_gc - ## ValueType: Boolean - ## Default: false - backoff_gc = false - - ## Backoff hibernation enabled - ## @doc overload_protection.backoff_hibernation - ## ValueType: Boolean - ## Default: true - backoff_hibernation = true - - ## Backoff hibernation enabled - ## @doc overload_protection.backoff_hibernation - ## ValueType: Boolean - ## Default: true - backoff_new_conn = true -} - -force_gc { - ## Force the MQTT connection process GC after this number of - ## messages or bytes passed through. - ## - ## @doc force_gc.enable - ## ValueType: Boolean - ## Default: true - enable = true - - ## GC the process after how many messages received - ## @doc force_gc.max_message_queue_len - ## ValueType: Integer - ## Range: (0, infinity) - ## Default: 16000 - count = 16000 - - ## GC the process after how much bytes passed through - ## - ## @doc force_gc.bytes - ## ValueType: Size - ## Default: 16MB - bytes = 16MB -} - -conn_congestion { - ## Whether to alarm the congested connections. - ## - ## Sometimes the mqtt connection (usually an MQTT subscriber) may - ## get "congested" because there're too many packets to sent. - ## The socket trys to buffer the packets until the buffer is - ## full. If more packets comes after that, the packets will be - ## "pending" in a queue and we consider the connection is - ## "congested". - ## - ## Enable this to send an alarm when there's any bytes pending in - ## the queue. You could set the `sndbuf` to a larger value if the - ## alarm is triggered too often. - ## - ## The name of the alarm is of format "conn_congestion//". - ## Where the is the client-id of the congested MQTT connection. - ## And the is the username or "unknown_user" of not provided by the client. - ## - ## @doc conn_congestion.enable_alarm - ## ValueType: Boolean - ## Default: true - enable_alarm = true - - ## Won't clear the congested alarm in how long time. - ## The alarm is cleared only when there're no pending bytes in - ## the queue, and also it has been `min_alarm_sustain_duration` - ## time since the last time we considered the connection is "congested". - ## - ## This is to avoid clearing and sending the alarm again too often. - ## - ## @doc conn_congestion.min_alarm_sustain_duration - ## ValueType: Duration - ## Default: 1m - min_alarm_sustain_duration = 1m -} - -rate_limit { - ## Maximum connections per second. - ## - ## @doc zones..max_conn_rate - ## ValueType: Number | infinity - ## Default: 1000 - ## Examples: - ## max_conn_rate: 1000 - max_conn_rate = 1000 - - ## Message limit for the a external MQTT connection. - ## - ## @doc rate_limit.conn_messages_in - ## ValueType: String | infinity - ## Default: infinity - ## Examples: 100 messages per 10 seconds. - ## conn_messages_in: "100,10s" - conn_messages_in = "100,10s" - - ## Limit the rate of receiving packets for a MQTT connection. - ## The rate is counted by bytes of packets per second. - ## - ## The connection won't accept more messages if the messages come - ## faster than the limit. - ## - ## @doc rate_limit.conn_bytes_in - ## ValueType: String | infinity - ## Default: infinity - ## Examples: 100KB incoming per 10 seconds. - ## conn_bytes_in: "100KB,10s" - ## - conn_bytes_in = "100KB,10s" -} - -quota { - ## Messages quota for the each of external MQTT connection. - ## This value consumed by the number of recipient on a message. - ## - ## @doc quota.conn_messages_routing - ## ValueType: String | infinity - ## Default: infinity - ## Examples: 100 messaegs per 1s: - ## quota.conn_messages_routing: "100,1s" - conn_messages_routing = "100,1s" - - ## Messages quota for the all of external MQTT connections. - ## This value consumed by the number of recipient on a message. - ## - ## @doc quota.overall_messages_routing - ## ValueType: String | infinity - ## Default: infinity - ## Examples: 200000 messages per 1s: - ## quota.overall_messages_routing: "200000,1s" - ## - overall_messages_routing = "200000,1s" -} - -##================================================================== -## Zones -##================================================================== -## A zone contains a set of configurations for listeners. -## -## A zone can be used by a listener via `listener...zone`. -## -## The configs defined in zones will override the global configs with the same key. -## -## For example given the following config: -## -## ``` -## a { -## b: 1, c: 1 -## } -## -## zone.my_zone { -## a { -## b:2 -## } -## } -## ``` -## -## The global config "a" is overridden by the configs "a" inside the zone "my_zone". -## If there is a listener uses the zone "my_zone", the value of config "a" will be: -## `{b:2, c: 1}`. -## Note that although the default value of `a.c` is `0`, the global value is used. -## i.e. configs in the zone have no default values. To overridde `a.c` we must configure -## it explicitly in the zone. -## -## All the global configs that can be overridden in zones are: -## - `stats.*` -## - `mqtt.*` -## - `authorization.*` -## - `flapping_detect.*` -## - `force_shutdown.*` -## - `conn_congestion.*` -## - `rate_limit.*` -## - `quota.*` -## - `force_gc.*` -## -## syntax: zones. -## example: zones.my_zone -zones.default { - -} - -##================================================================== -## Broker -##================================================================== -broker { - ## System interval of publishing $SYS messages. - ## - ## @doc broker.sys_msg_interval - ## ValueType: Duration | disabled - ## Default: 1m - sys_msg_interval = 1m - - ## System heartbeat interval of publishing following heart beat message: - ## - "$SYS/brokers//uptime" - ## - "$SYS/brokers//datetime" - ## - ## @doc broker.sys_heartbeat_interval - ## ValueType: Duration - ## Default: 30s | disabled - sys_heartbeat_interval = 30s - - ## Session locking strategy in a cluster. - ## - ## @doc broker.session_locking_strategy - ## ValueType: local | one | quorum | all - ## - local: only lock the session locally on the current node - ## - one: select only one remove node to lock the session - ## - quorum: select some nodes to lock the session - ## - all: lock the session on all of the nodes in the cluster - ## Default: quorum - session_locking_strategy = quorum - - ## Dispatch strategy for shared subscription - ## - ## @doc broker.shared_subscription_strategy - ## ValueType: random | round_robin | sticky | hash - ## - random: dispatch the message to a random selected subscriber - ## - round_robin: select the subscribers in a round-robin manner - ## - sticky: always use the last selected subscriber to dispatch, - ## until the susbcriber disconnected. - ## - hash: select the subscribers by the hash of clientIds - ## Default: round_robin - shared_subscription_strategy = round_robin - - ## Enable/disable shared dispatch acknowledgement for QoS1 and QoS2 messages - ## This should allow messages to be dispatched to a different subscriber in - ## the group in case the picked (based on shared_subscription_strategy) one # is offline - ## - ## @doc broker.shared_dispatch_ack_enabled - ## ValueType: Boolean - ## Default: false - shared_dispatch_ack_enabled = false - - ## Enable batch clean for deleted routes. - ## - ## @doc broker.route_batch_clean - ## ValueType: Boolean - ## Default: true - route_batch_clean = true - - ## Performance toggle for subscribe/unsubscribe wildcard topic. - ## Change this toggle only when there are many wildcard topics. - ## - ## NOTE: when changing from/to 'global' lock, it requires all - ## nodes in the cluster to be stopped before the change. - ## - ## @doc broker.perf.route_lock_type - ## ValueType: key | tab | global - ## - key: mnesia translational updates with per-key locks. recommended for single node setup. - ## - tab: mnesia translational updates with table lock. recommended for multi-nodes setup. - ## - global: global lock protected updates. recommended for larger cluster. - ## Default: key - perf.route_lock_type = key - - ## Enable trie path compaction. - ## Enabling it significantly improves wildcard topic subscribe - ## rate, if wildcard topics have unique prefixes like: - ## 'sensor//+/', where ID is unique per subscriber. - ## - ## Topic match performance (when publishing) may degrade if messages - ## are mostly published to topics with large number of levels. - ## - ## NOTE: This is a cluster-wide configuration. - ## It requires all nodes to be stopped before changing it. - ## - ## @doc broker.perf.trie_compaction - ## ValueType: Boolean - ## Default: true - perf.trie_compaction = true -} - -##================================================================== -## System Monitor -##================================================================== -sysmon { - ## The time interval for the periodic process limit check - ## - ## @doc sysmon.vm.process_check_interval - ## ValueType: Duration - ## Default: 30s - vm.process_check_interval = 30s - - ## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is set. - ## - ## @doc sysmon.vm.process_high_watermark - ## ValueType: Percentage - ## Default: 80% - vm.process_high_watermark = 80% - - ## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is clear. - ## - ## @doc sysmon.vm.process_low_watermark - ## ValueType: Percentage - ## Default: 60% - vm.process_low_watermark = 60% - - ## Enable Long GC monitoring. - ## Notice: don't enable the monitor in production for: - ## https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421 - ## - ## @doc sysmon.vm.long_gc - ## ValueType: Duration | disabled - ## Default: disabled - vm.long_gc = disabled - - ## Enable Long Schedule(ms) monitoring. - ## - ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 - ## - ## @doc sysmon.vm.long_schedule - ## ValueType: Duration | disabled - ## Default: disabled - vm.long_schedule = 240ms - - ## Enable Large Heap monitoring. - ## - ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 - ## - ## @doc sysmon.vm.large_heap - ## ValueType: Size | disabled - ## Default: 32MB - vm.large_heap = 32MB - - ## Enable Busy Port monitoring. - ## - ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 - ## - ## @doc sysmon.vm.busy_port - ## ValueType: Boolean - ## Default: true - vm.busy_port = true - - ## Enable Busy Dist Port monitoring. - ## - ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 - ## - ## @doc sysmon.vm.busy_dist_port - ## ValueType: Boolean - ## Default: true - vm.busy_dist_port = true - - ## The time interval for the periodic cpu check - ## - ## @doc sysmon.os.cpu_check_interval - ## ValueType: Duration - ## Default: 60s - os.cpu_check_interval = 60s - - ## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is set. - ## - ## @doc sysmon.os.cpu_high_watermark - ## ValueType: Percentage - ## Default: 80% - os.cpu_high_watermark = 80% - - ## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is clear. - ## - ## @doc sysmon.os.cpu_low_watermark - ## ValueType: Percentage - ## Default: 60% - os.cpu_low_watermark = 60% - - ## The time interval for the periodic memory check - ## - ## @doc sysmon.os.mem_check_interval - ## ValueType: Duration | disabled - ## Default: 60s - os.mem_check_interval = 60s - - ## The threshold, as percentage of system memory, for how much system memory can be allocated before the corresponding alarm is set. - ## - ## @doc sysmon.os.sysmem_high_watermark - ## ValueType: Percentage - ## Default: 70% - os.sysmem_high_watermark = 70% - - ## The threshold, as percentage of system memory, for how much system memory can be allocated by one Erlang process before the corresponding alarm is set. - ## - ## @doc sysmon.os.procmem_high_watermark - ## ValueType: Percentage - ## Default: 5% - os.procmem_high_watermark = 5% -} - -##================================================================== -## Alarm -##================================================================== -alarm { - ## Specifies the actions to take when an alarm is activated - ## - ## @doc alarm.actions - ## ValueType: Array - ## Default: [log, publish] - actions = [log, publish] - - ## The maximum number of deactivated alarms - ## - ## @doc alarm.size_limit - ## ValueType: Integer - ## Default: 1000 - size_limit = 1000 - - ## Validity Period of deactivated alarms - ## - ## @doc alarm.validity_period - ## ValueType: Duration - ## Default: 24h - validity_period = 24h -} - -## Config references for listeners - -## Socket options for TCP connections -## See: http://erlang.org/doc/man/inet.html -example_common_tcp_options { - ## Specify the {active, N} option for this Socket. - ## - ## See: https://erlang.org/doc/man/inet.html#setopts-2 - ## - ## @doc listeners..tcp.active_n - ## ValueType: Number - ## Default: 100 - tcp.active_n = 100 - - ## TCP backlog defines the maximum length that the queue of - ## pending connections can grow to. - ## - ## @doc listeners..tcp.backlog - ## ValueType: Number - ## Range: [0, 1048576] - ## Default: 1024 - tcp.backlog = 1024 - - ## The TCP send timeout for the connections. - ## - ## @doc listeners..tcp.send_timeout - ## ValueType: Duration - ## Default: 15s - tcp.send_timeout = 15s - - ## Close the connection if send timeout. - ## - ## @doc listeners..tcp.send_timeout_close - ## ValueType: Boolean - ## Default: true - tcp.send_timeout_close = true - - ## The TCP receive buffer(os kernel) for the connections. - ## - ## @doc listeners..tcp.recbuf - ## ValueType: Size - ## Default: notset - #tcp.recbuf: 2KB - - ## The TCP send buffer(os kernel) for the connections. - ## - ## @doc listeners..tcp.sndbuf - ## ValueType: Size - ## Default: notset - #tcp.sndbuf: 4KB - - ## The size of the user-level software buffer used by the driver. - ## - ## @doc listeners..tcp.buffer - ## ValueType: Size - ## Default: notset - #tcp.buffer: 4KB - - ## The socket is set to a busy state when the amount of data queued internally - ## by the ERTS socket implementation reaches this limit. - ## - ## @doc listeners..tcp.high_watermark - ## ValueType: Size - ## Default: 1MB - tcp.high_watermark = 1MB - - ## The TCP_NODELAY flag for the connections. - ## - ## @doc listeners..tcp.nodelay - ## ValueType: Boolean - ## Default: false - tcp.nodelay = false - - ## The SO_REUSEADDR flag for the connections. - ## - ## @doc listeners..tcp.reuseaddr - ## ValueType: Boolean - ## Default: true - tcp.reuseaddr = true -} - -## Socket options for SSL connections -## See: http://erlang.org/doc/man/ssl.html -example_common_ssl_options { - - ## A performance optimization setting, it allows clients to reuse - ## pre-existing sessions, instead of initializing new ones. - ## Read more about it here. - ## - ## @doc listeners..ssl.reuse_sessions - ## ValueType: Boolean - ## Default: true - ssl.reuse_sessions = true - - ## SSL parameter renegotiation is a feature that allows a client and a server - ## to renegotiate the parameters of the SSL connection on the fly. - ## RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, - ## you drop support for the insecure renegotiation, prone to MitM attacks. - ## - ## @doc listeners..ssl.secure_renegotiate - ## ValueType: Boolean - ## Default: true - ssl.secure_renegotiate = true - - ## In protocols that support client-initiated renegotiation, - ## the cost of resources of such an operation is higher for the server than the client. - ## This can act as a vector for denial of service attacks. - ## The SSL application already takes measures to counter-act such attempts, - ## but client-initiated renegotiation can be strictly disabled by setting this option to false. - ## The default value is true. Note that disabling renegotiation can result in - ## long-lived connections becoming unusable due to limits on - ## the number of messages the underlying cipher suite can encipher. - ssl.client_renegotiation = true - - ## An important security setting, it forces the cipher to be set based - ## on the server-specified order instead of the client-specified order, - ## hence enforcing the (usually more properly configured) security - ## ordering of the server administrator. - ## - ## @doc listeners..ssl.honor_cipher_order - ## ValueType: Boolean - ## Default: true - ssl.honor_cipher_order = true - - # ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] - # TLS 1.3: "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256" - # TLS 1-1.2 "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" - # PSK: "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" - # NOTE: If PSK cipher-suites are intended, tlsv1.3 should not be enabled in 'versions' config - # NOTE: by default, ALL ciphers are enabled - # ssl.ciphers = "" - - ## TLS Handshake timeout. - ## - ## @doc listeners..ssl.handshake_timeout - ## ValueType: Duration - ## Default: 15s - ssl.handshake_timeout = 15s - - ## Maximum number of non-self-issued intermediate certificates that - ## can follow the peer certificate in a valid certification path. - ## - ## @doc listeners..ssl.depth - ## ValueType: Integer - ## Default: 10 - ssl.depth = 10 - - ## Path to the file containing the user's private PEM-encoded key. - ## - ## @doc listeners..ssl.keyfile - ## ValueType: File - ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem" - ssl.keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem" - - ## Path to a file containing the user certificate. - ## - ## @doc listeners..ssl.certfile - ## ValueType: File - ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem" - ssl.certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem" - - ## Path to the file containing PEM-encoded CA certificates. The CA certificates - ## are used during server authentication and when building the client certificate chain. - ## - ## @doc listeners..ssl.cacertfile - ## ValueType: File - ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem" - ssl.cacertfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem" - - ## Maximum number of non-self-issued intermediate certificates that - ## can follow the peer certificate in a valid certification path. - ## - ## @doc listeners..ssl.depth - ## ValueType: Number - ## Default: 10 - ssl.depth = 10 - - ## String containing the user's password. Only used if the private keyfile - ## is password-protected. - ## - ## See: listener.ssl.$name.key_password - ## - ## @doc listeners..ssl.depth - ## ValueType: String - ## Default: "" - #ssl.key_password: "" - - ## The Ephemeral Diffie-Helman key exchange is a very effective way of - ## ensuring Forward Secrecy by exchanging a set of keys that never hit - ## the wire. Since the DH key is effectively signed by the private key, - ## it needs to be at least as strong as the private key. In addition, - ## the default DH groups that most of the OpenSSL installations have - ## are only a handful (since they are distributed with the OpenSSL - ## package that has been built for the operating system it’s running on) - ## and hence predictable (not to mention, 1024 bits only). - ## In order to escape this situation, first we need to generate a fresh, - ## strong DH group, store it in a file and then use the option above, - ## to force our SSL application to use the new DH group. Fortunately, - ## OpenSSL provides us with a tool to do that. Simply run: - ## openssl dhparam -out dh-params.pem 2048 - ## - ## @doc listeners..ssl.dhfile - ## ValueType: File - ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/dh-params.pem" - #ssl.dhfile: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/dh-params.pem" - - ## A server only does x509-path validation in mode verify_peer, - ## as it then sends a certificate request to the client (this - ## message is not sent if the verify option is verify_none). - ## You can then also want to specify option fail_if_no_peer_cert. - ## More information at: http://erlang.org/doc/man/ssl.html - ## - ## @doc listeners..ssl.verify - ## ValueType: verify_peer | verify_none - ## Default: verify_none - ssl.verify = verify_none - - ## Used together with {verify, verify_peer} by an SSL server. If set to true, - ## the server fails if the client does not have a certificate to send, that is, - ## sends an empty certificate. - ## - ## @doc listeners..ssl.fail_if_no_peer_cert - ## ValueType: Boolean - ## Default: true - ssl.fail_if_no_peer_cert = false - -} - -## Socket options for websocket connections -example_common_websocket_options { - ## The path of WebSocket MQTT endpoint - ## - ## @doc listeners..websocket.mqtt_path - ## ValueType: Path - ## Default: "/mqtt" - websocket.mqtt_path = "/mqtt" - - ## Whether a WebSocket message is allowed to contain multiple MQTT packets - ## - ## @doc listeners..websocket.mqtt_piggyback - ## ValueType: single | multiple - ## Default: multiple - websocket.mqtt_piggyback = multiple - - ## The compress flag for external WebSocket connections. - ## - ## If this Value is set true,the websocket message would be compressed - ## - ## @doc listeners..websocket.compress - ## ValueType: Boolean - ## Default: false - websocket.compress = false - - ## The idle timeout for external WebSocket connections. - ## - ## @doc listeners..websocket.idle_timeout - ## ValueType: Duration | infinity - ## Default: infinity - websocket.idle_timeout = infinity - - ## The max frame size for external WebSocket connections. - ## - ## @doc listeners..websocket.max_frame_size - ## ValueType: Size - ## Default: infinity - websocket.max_frame_size = infinity - - ## If set to true, the server fails if the client does not - ## have a Sec-WebSocket-Protocol to send. - ## Set to false for WeChat MiniApp. - ## - ## @doc listeners..websocket.fail_if_no_subprotocol - ## ValueType: Boolean - ## Default: true - websocket.fail_if_no_subprotocol = true - - ## Supported subprotocols - ## - ## @doc listeners..websocket.supported_subprotocols - ## ValueType: String - ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 - websocket.supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" - - ## Enable origin check in header for websocket connection - ## - ## @doc listeners..websocket.check_origin_enable - ## ValueType: Boolean - ## Default: false - websocket.check_origin_enable = false - - ## Allow origin to be absent in header in websocket connection - ## when check_origin_enable is true - ## - ## @doc listeners..websocket.allow_origin_absence - ## ValueType: Boolean - ## Default: true - websocket.allow_origin_absence = true - - ## Comma separated list of allowed origin in header for websocket connection - ## - ## @doc listeners..websocket.check_origins - ## ValueType: String - ## Examples: - ## local http dashboard url - ## check_origins: "http://localhost:18083, http://127.0.0.1:18083" - ## Default: "" - websocket.check_origins = "http://localhost:18083, http://127.0.0.1:18083" - - ## Specify which HTTP header for real source IP if the EMQ X cluster is - ## deployed behind NGINX or HAProxy. - ## - ## @doc listeners..websocket.proxy_address_header - ## ValueType: String - ## Default: X-Forwarded-For - websocket.proxy_address_header = X-Forwarded-For - - ## Specify which HTTP header for real source port if the EMQ X cluster is - ## deployed behind NGINX or HAProxy. - ## - ## @doc listeners..websocket.proxy_port_header - ## ValueType: String - ## Default: X-Forwarded-Port - websocket.proxy_port_header = X-Forwarded-Port - - websocket.deflate_opts { - ## The level of deflate options for external WebSocket connections. - ## - ## @doc listeners..websocket.deflate_opts.level - ## ValueType: none | default | best_compression | best_speed - ## Default: default - level = default - - ## The mem_level of deflate options for external WebSocket connections. - ## - ## @doc listeners..websocket.deflate_opts.mem_level - ## ValueType: Integer - ## Range: [1,9] - ## Default: 8 - mem_level = 8 - - ## The strategy of deflate options for external WebSocket connections. - ## - ## @doc listeners..websocket.deflate_opts.strategy - ## ValueType: default | filtered | huffman_only | rle - ## Default: default - strategy = default - - ## The deflate option for external WebSocket connections. - ## - ## @doc listeners..websocket.deflate_opts.server_context_takeover - ## ValueType: takeover | no_takeover - ## Default: takeover - server_context_takeover = takeover - - ## The deflate option for external WebSocket connections. - ## - ## @doc listeners..websocket.deflate_opts.client_context_takeover - ## ValueType: takeover | no_takeover - ## Default: takeover - client_context_takeover = takeover - - ## The deflate options for external WebSocket connections. - ## - ## - ## @doc listeners..websocket.deflate_opts.server_max_window_bits - ## ValueType: Integer - ## Range: [8,15] - ## Default: 15 - server_max_window_bits = 15 - - ## The deflate options for external WebSocket connections. - ## - ## @doc listeners..websocket.deflate_opts.client_max_window_bits - ## ValueType: Integer - ## Range: [8,15] - ## Default: 15 - client_max_window_bits = 15 - } -} - -persistent_session_store { - ## Enable/disable internal persistent session store. - ## - ## @doc persistent_session_store.enabled - ## ValueType: Boolean - ## Default: false - enabled = false - - ## How long are undelivered messages retained in the store - ## - ## @doc persistent_session_store.max_retain_undelivered - ## ValueType: Duration - ## Default: 1h - max_retain_undelivered = 1h - - ## The time interval in which to try to run garbage collection of persistent session messages - ## - ## @doc persistent_session_store.message_gc_interval - ## ValueType: Duration - ## Default: 1h - message_gc_interval = 1h - - ## The time interval in which to try to run garbage collection of persistent session transient data - ## - ## @doc persistent_session_store.session_message_gc_interval - ## ValueType: Duration - ## Default: 1m - session_message_gc_interval = 1m -} diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index ddd4349bb..2398a7dee 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -63,12 +63,12 @@ %% structured logging, meta is for handler's filter. -define(SLOG(Level, Data, Meta), -%% check 'allow' here, only evaluate Data when necessary +%% check 'allow' here, only evaluate Data and Meta when necessary case logger:allow(Level, ?MODULE) of true -> - logger:log(Level, (Data), Meta#{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} + logger:log(Level, (Data), (Meta#{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} , line => ?LINE - }); + })); false -> ok end). diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index d0cb07a68..290299aee 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -550,7 +550,6 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) -> {error, Rc = ?RC_NOT_AUTHORIZED, NChannel} -> ?SLOG(warning, #{ msg => "cannot_publish_to_topic", - topic => Topic, reason => emqx_reason_codes:name(Rc) }, #{topic => Topic}), case emqx:get_config([authorization, deny_action], ignore) of @@ -568,7 +567,6 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) -> {error, Rc = ?RC_QUOTA_EXCEEDED, NChannel} -> ?SLOG(warning, #{ msg => "cannot_publish_to_topic", - topic => Topic, reason => emqx_reason_codes:name(Rc) }, #{topic => Topic}), case QoS of diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index c44cfe15e..a2101efa1 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -455,8 +455,7 @@ kick_session(Action, ClientId, ChanPid) -> kick_session(ClientId) -> case lookup_channels(ClientId) of [] -> - ?SLOG(warning, #{msg => "kicked_an_unknown_session", - clientid => ClientId}, + ?SLOG(warning, #{msg => "kicked_an_unknown_session"}, #{clientid => unicode:characters_to_list(ClientId, utf8)}), ok; ChanPids -> diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index cb3da361a..df7c7523f 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -118,7 +118,6 @@ handle_cast({detected, #flapping{clientid = ClientId, true -> %% Flapping happened:( ?SLOG(warning, #{ msg => "flapping_detected", - client_id => ClientId, peer_host => fmt_host(PeerHost), detect_cnt => DetectCnt, wind_time_in_ms => WindTime @@ -134,7 +133,6 @@ handle_cast({detected, #flapping{clientid = ClientId, false -> ?SLOG(warning, #{ msg => "client_disconnected", - client_id => ClientId, peer_host => fmt_host(PeerHost), detect_cnt => DetectCnt, interval => Interval diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl index d1fbd20b8..f8a5f3828 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl @@ -35,7 +35,6 @@ , filter_topic/2 , filter_ip_address/2 ]). --export([template/1]). -export([handler_id/2]). -export([payload_encode/0]). @@ -158,20 +157,6 @@ formatter(#{type := _Type}) -> } }. -%% Don't log clientid since clientid only supports exact match, all client ids are the same. -%% if clientid is not latin characters. the logger_formatter restricts the output must be `~tp` -%% (actually should use `~ts`), the utf8 characters clientid will become very difficult to read. -template(clientid) -> - [time, " [", level, "] ", {peername, [peername, " "], []}, msg, "\n"]; -template(_) -> - [time, " [", level, "] ", - {clientid, - [{peername, [clientid, "@", peername, " "], [clientid, " "]}], - [{peername, [peername, " "], []}] - }, - msg, "\n" - ]. - filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) -> Init = #{id => Id, level => Level, dst => Dst}, case Filters of From db9d26903435d6e4372489f697bbdc19e55da868 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 30 Dec 2021 10:29:20 +0800 Subject: [PATCH 36/99] fix(dashboard): batter auth failed response message --- apps/emqx_dashboard/src/emqx_dashboard.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index a85db870c..e79b0210c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -151,9 +151,9 @@ authorize(Req) -> ok -> ok; {error, token_timeout} -> - {401, <<"TOKEN_TIME_OUT">>, <<"POST '/login', get new token">>}; + {401, <<"TOKEN_TIME_OUT">>, <<"Token expired, get new token by POST /login">>}; {error, not_found} -> - {401, <<"BAD_TOKEN">>, <<"POST '/login', get new token">>} + {401, <<"BAD_TOKEN">>, <<"Get a token by POST /login">>} end; _ -> return_unauthorized(<<"AUTHORIZATION_HEADER_ERROR">>, From d60c586bfb7613ec645b2c0036db065d824a6d1f Mon Sep 17 00:00:00 2001 From: lafirest Date: Thu, 30 Dec 2021 10:00:43 +0800 Subject: [PATCH 37/99] fix(emqx_slow_subs): limit the max size of top-k table --- apps/emqx_management/src/emqx_mgmt_api.erl | 1 + apps/emqx_slow_subs/include/emqx_slow_subs.hrl | 2 ++ apps/emqx_slow_subs/src/emqx_slow_subs.erl | 3 ++- apps/emqx_slow_subs/src/emqx_slow_subs_api.erl | 6 +++++- apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl | 2 +- apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl | 2 +- 6 files changed, 12 insertions(+), 4 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index a2450f988..256d31027 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -30,6 +30,7 @@ -export([ node_query/5 , cluster_query/4 , select_table_with_count/5 + , b2i/1 ]). -export([do_query/6]). diff --git a/apps/emqx_slow_subs/include/emqx_slow_subs.hrl b/apps/emqx_slow_subs/include/emqx_slow_subs.hrl index 0b5e3a035..bfdfcc22f 100644 --- a/apps/emqx_slow_subs/include/emqx_slow_subs.hrl +++ b/apps/emqx_slow_subs/include/emqx_slow_subs.hrl @@ -18,6 +18,8 @@ -define(INDEX(Latency, ClientId), {Latency, ClientId}). +-define(MAX_TAB_SIZE, 1000). + -record(top_k, { index :: index() , type :: emqx_message_latency_stats:latency_type() , last_update_time :: pos_integer() diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.erl b/apps/emqx_slow_subs/src/emqx_slow_subs.erl index acb4ea441..50de524af 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs.erl @@ -251,7 +251,8 @@ publish(TickTime, Notices) -> ok. load() -> - MaxSize = emqx:get_config([emqx_slow_subs, top_k_num]), + MaxSizeT = emqx:get_config([emqx_slow_subs, top_k_num]), + MaxSize = erlang:min(MaxSizeT, ?MAX_TAB_SIZE), _ = emqx:hook('message.slow_subs_stats', {?MODULE, on_stats_update, [#{max_size => MaxSize}]} ), diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl index 8af4f14ea..4c5873ab9 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl @@ -87,7 +87,11 @@ slow_subs(delete, _) -> ok = emqx_slow_subs:clear_history(), {204}; -slow_subs(get, #{query_string := QS}) -> +slow_subs(get, #{query_string := QST}) -> + LimitT = maps:get(<<"limit">>, QST, ?MAX_TAB_SIZE), + Limit = erlang:min(?MAX_TAB_SIZE, emqx_mgmt_api:b2i(LimitT)), + Page = maps:get(<<"page">>, QST, 1), + QS = QST#{<<"limit">> => Limit, <<"page">> => Page}, Data = emqx_mgmt_api:paginate({?TOPK_TAB, [{traverse, last_prev}]}, QS, ?FORMAT_FUN), {200, Data}. diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl index c187a091e..2cef9affc 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl @@ -14,7 +14,7 @@ fields("emqx_slow_subs") -> "The latency threshold for statistics, the minimum value is 100ms")} , {expire_interval, sc(emqx_schema:duration_ms(), - "5m", + "300s", "The eviction time of the record, which in the statistics record table")} , {top_k_num, sc(integer(), diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl index f66122775..3745ffe04 100644 --- a/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl +++ b/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl @@ -90,7 +90,7 @@ t_log_and_pub(_) -> ?assert(RecSum >= 5), ?assert(lists:all(fun(E) -> E =< 3 end, Recs)), - timer:sleep(2000), + timer:sleep(3000), ?assert(ets:info(?TOPK_TAB, size) =:= 0), [Client ! stop || Client <- Clients], ok. From ed086728cff867682b007f28dba82556a18d3325 Mon Sep 17 00:00:00 2001 From: lafirest Date: Thu, 30 Dec 2021 11:32:08 +0800 Subject: [PATCH 38/99] fix: Revert "fix(emqx_retainer): add support for RAP falg" This reverts commit b80a01554b14442d777a7972aeac519d0afeb90e. --- apps/emqx_retainer/src/emqx_retainer.erl | 48 +++++++++++------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 24dd05524..9be449b60 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -27,10 +27,10 @@ , on_message_publish/2 ]). --export([ dispatch/5 +-export([ dispatch/4 , delete_message/2 , store_retained/2 - , deliver/6]). + , deliver/5]). -export([ get_expiry_time/1 , update_config/1 @@ -78,7 +78,7 @@ on_session_subscribed(_, _, #{share := ShareName}, _) when ShareName =/= undefin on_session_subscribed(_, Topic, #{rh := Rh} = Opts, Context) -> IsNew = maps:get(is_new, Opts, true), case Rh =:= 0 orelse (Rh =:= 1 andalso IsNew) of - true -> dispatch(Context, Topic, Opts); + true -> dispatch(Context, Topic); _ -> ok end. @@ -111,26 +111,26 @@ on_message_publish(Msg, _) -> start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). --spec dispatch(context(), pid(), topic(), emqx_types:subopts(), cursor()) -> ok. -dispatch(Context, Pid, Topic, Opts, Cursor) -> +-spec dispatch(context(), pid(), topic(), cursor()) -> ok. +dispatch(Context, Pid, Topic, Cursor) -> Mod = get_backend_module(), case Cursor =/= undefined orelse emqx_topic:wildcard(Topic) of false -> {ok, Result} = Mod:read_message(Context, Topic), - deliver(Result, Context, Pid, Topic, Opts, undefined); + deliver(Result, Context, Pid, Topic, undefined); true -> {ok, Result, NewCursor} = Mod:match_messages(Context, Topic, Cursor), - deliver(Result, Context, Pid, Topic, Opts, NewCursor) + deliver(Result, Context, Pid, Topic, NewCursor) end. -deliver([], Context, Pid, Topic, Opts, Cursor) -> +deliver([], Context, Pid, Topic, Cursor) -> case Cursor of undefined -> ok; _ -> - dispatch(Context, Pid, Topic, Opts, Cursor) + dispatch(Context, Pid, Topic, Cursor) end; -deliver(Result, #{context_id := Id} = Context, Pid, Topic, Opts, Cursor) -> +deliver(Result, #{context_id := Id} = Context, Pid, Topic, Cursor) -> case erlang:is_process_alive(Pid) of false -> ok; @@ -138,12 +138,12 @@ deliver(Result, #{context_id := Id} = Context, Pid, Topic, Opts, Cursor) -> #{msg_deliver_quota := MaxDeliverNum} = emqx:get_config([?APP, flow_control]), case MaxDeliverNum of 0 -> - _ = [Pid ! {deliver, Topic, handle_retain_opts(Opts, Msg)} || Msg <- Result], + _ = [Pid ! {deliver, Topic, Msg} || Msg <- Result], ok; _ -> - case do_deliver(Result, Id, Pid, Topic, Opts) of + case do_deliver(Result, Id, Pid, Topic) of ok -> - deliver([], Context, Pid, Topic, Opts, Cursor); + deliver([], Context, Pid, Topic, Cursor); abort -> ok end @@ -280,9 +280,9 @@ is_too_big(Size) -> Limit > 0 andalso (Size > Limit). %% @private -dispatch(Context, Topic, Opts) -> - emqx_retainer_pool:async_submit(fun ?MODULE:dispatch/5, - [Context, self(), Topic, Opts, undefined]). +dispatch(Context, Topic) -> + emqx_retainer_pool:async_submit(fun ?MODULE:dispatch/4, + [Context, self(), Topic, undefined]). -spec delete_message(context(), topic()) -> ok. delete_message(Context, Topic) -> @@ -305,16 +305,16 @@ clean(Context) -> Mod = get_backend_module(), Mod:clean(Context). --spec do_deliver(list(term()), pos_integer(), pid(), topic(), emqx_types:subopts()) -> ok | abort. -do_deliver([Msg | T], Id, Pid, Topic, Opts) -> +-spec do_deliver(list(term()), pos_integer(), pid(), topic()) -> ok | abort. +do_deliver([Msg | T], Id, Pid, Topic) -> case require_semaphore(?DELIVER_SEMAPHORE, Id) of true -> - Pid ! {deliver, Topic, handle_retain_opts(Opts, Msg)}, - do_deliver(T, Id, Pid, Topic, Opts); + Pid ! {deliver, Topic, Msg}, + do_deliver(T, Id, Pid, Topic); _ -> abort end; -do_deliver([], _, _, _, _) -> +do_deliver([], _, _, _) -> ok. -spec require_semaphore(semaphore(), pos_integer()) -> boolean(). @@ -484,9 +484,3 @@ load(Context) -> unload() -> emqx:unhook('message.publish', {?MODULE, on_message_publish}), emqx:unhook('session.subscribed', {?MODULE, on_session_subscribed}). - -handle_retain_opts(#{rap := 0}, Message) -> - emqx_message:set_header(retain, false, Message); - -handle_retain_opts(_, Message) -> - Message. From f466a9f2dedb065ca92fcf71ac0e1af6afd044b9 Mon Sep 17 00:00:00 2001 From: lafirest Date: Thu, 30 Dec 2021 11:26:24 +0800 Subject: [PATCH 39/99] fix(emqx_slow_subs): fix timer management error --- apps/emqx_slow_subs/src/emqx_slow_subs.erl | 60 ++++++++++++++-------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.erl b/apps/emqx_slow_subs/src/emqx_slow_subs.erl index 50de524af..445465e1f 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs.erl @@ -39,6 +39,8 @@ -type state() :: #{ enable := boolean() , last_tick_at := pos_integer() + , expire_timer := undefined | reference() + , notice_timer := undefined | reference() }. -type log() :: #{ rank := pos_integer() @@ -141,8 +143,14 @@ init_topk_tab() -> %%-------------------------------------------------------------------- init([]) -> + InitState = #{enable => false, + last_tick_at => 0, + expire_timer => undefined, + notice_timer => undefined + }, + Enable = emqx:get_config([emqx_slow_subs, enable]), - {ok, check_enable(Enable, #{enable => false})}. + {ok, check_enable(Enable, InitState)}. handle_call({update_settings, Enable}, _From, State) -> State2 = check_enable(Enable, State), @@ -161,23 +169,23 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info(expire_tick, State) -> - expire_tick(), Logs = ets:tab2list(?TOPK_TAB), do_clear(Logs), - {noreply, State}; + State1 = start_timer(expire_timer, fun expire_tick/0, State), + {noreply, State1}; handle_info(notice_tick, State) -> - notice_tick(), Logs = ets:tab2list(?TOPK_TAB), do_notification(Logs, State), - {noreply, State#{last_tick_at := ?NOW}}; + State1 = start_timer(notice_timer, fun notice_tick/0, State), + {noreply, State1#{last_tick_at := ?NOW}}; handle_info(Info, State) -> ?LOG(error, "Unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, _) -> - unload(), +terminate(_Reason, State) -> + _ = unload(State), ok. code_change(_OldVsn, State, _Extra) -> @@ -191,10 +199,9 @@ expire_tick() -> notice_tick() -> case emqx:get_config([emqx_slow_subs, notice_interval]) of - 0 -> ok; + 0 -> undefined; Interval -> - erlang:send_after(Interval, self(), ?FUNCTION_NAME), - ok + erlang:send_after(Interval, self(), ?FUNCTION_NAME) end. -spec do_notification(list(), state()) -> ok. @@ -250,16 +257,23 @@ publish(TickTime, Notices) -> _ = emqx_broker:safe_publish(Msg), ok. -load() -> +load(State) -> MaxSizeT = emqx:get_config([emqx_slow_subs, top_k_num]), MaxSize = erlang:min(MaxSizeT, ?MAX_TAB_SIZE), _ = emqx:hook('message.slow_subs_stats', {?MODULE, on_stats_update, [#{max_size => MaxSize}]} ), - ok. -unload() -> - emqx:unhook('message.slow_subs_stats', {?MODULE, on_stats_update}). + State1 = start_timer(notice_timer, fun notice_tick/0, State), + State2 = start_timer(expire_timer, fun expire_tick/0, State1), + State2#{enable := true, last_tick_at => ?NOW}. + + +unload(#{notice_timer := NoticeTimer, expire_timer := ExpireTimer} = State) -> + emqx:unhook('message.slow_subs_stats', {?MODULE, on_stats_update}), + State#{notice_timer := cancel_timer(NoticeTimer), + expire_timer := cancel_timer(ExpireTimer) + }. do_clear(Logs) -> Now = ?NOW, @@ -304,16 +318,22 @@ check_enable(Enable, #{enable := IsEnable} = State) -> IsEnable -> State; true -> - notice_tick(), - expire_tick(), - load(), - State#{enable := true, last_tick_at => ?NOW}; + load(State); _ -> - unload(), - State#{enable := false} + unload(State) end. update_threshold() -> Threshold = emqx:get_config([emqx_slow_subs, threshold]), emqx_message_latency_stats:update_threshold(Threshold), ok. + +start_timer(Name, Fun, State) -> + _ = cancel_timer(maps:get(Name, State)), + State#{Name := Fun()}. + +cancel_timer(TimerRef) when is_reference(TimerRef) -> + _ = erlang:cancel_timer(TimerRef), + undefined; +cancel_timer(_) -> + undefined. From 39c29b2396ebb42b2c69d8312cd55247e128e6f8 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 29 Dec 2021 16:12:14 +0800 Subject: [PATCH 40/99] fix(connector): redis cluster `servers` field --- .../src/emqx_connector_redis.erl | 39 +++++++++++++++---- .../src/emqx_dashboard_swagger.erl | 2 + 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 075ede0bc..2b71ab808 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -20,12 +20,19 @@ -include_lib("emqx/include/logger.hrl"). -type server() :: tuple(). - +%% {"127.0.0.1", 7000} +%% For eredis:start_link/1~7 -reflect_type([server/0]). - -typerefl_from_string({server/0, ?MODULE, to_server}). --export([to_server/1]). +-type servers() :: list(). +%% [{"127.0.0.1", 7000}, {"127.0.0.2", 7000}] +%% For eredis_cluster +-reflect_type([servers/0]). +-typerefl_from_string({servers/0, ?MODULE, to_servers}). + +-export([ to_server/1 + , to_servers/1]). -export([roots/0, fields/1]). @@ -63,14 +70,14 @@ fields(single) -> redis_fields() ++ emqx_connector_schema_lib:ssl_fields(); fields(cluster) -> - [ {servers, #{type => hoconsc:array(server())}} + [ {servers, #{type => servers()}} , {redis_type, #{type => hoconsc:enum([cluster]), default => cluster}} ] ++ redis_fields() ++ emqx_connector_schema_lib:ssl_fields(); fields(sentinel) -> - [ {servers, #{type => hoconsc:array(server())}} + [ {servers, #{type => servers()}} , {redis_type, #{type => hoconsc:enum([sentinel]), default => sentinel}} , {sentinel, #{type => string()}} @@ -181,7 +188,23 @@ redis_fields() -> ]. to_server(Server) -> - case string:tokens(Server, ":") of - [Host, Port] -> {ok, {Host, list_to_integer(Port)}}; - _ -> {error, Server} + try {ok, parse_server(Server)} + catch + throw : Error -> + Error + end. + +to_servers(Servers) -> + try {ok, lists:map(fun parse_server/1, string:tokens(Servers, ", "))} + catch + throw : _Reason -> + {error, Servers} + end. + +parse_server(Server) -> + case string:tokens(Server, ": ") of + [Host, Port] -> + {Host, list_to_integer(Port)}; + _ -> + throw({error, Server}) end. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 9a54be9c5..36f50e081 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -423,8 +423,10 @@ typename_to_spec("duration_ms()", _Mod) -> #{type => string, example => <<"32s"> typename_to_spec("percent()", _Mod) -> #{type => number, example => <<"12%">>}; typename_to_spec("file()", _Mod) -> #{type => string, example => <<"/path/to/file">>}; typename_to_spec("ip_port()", _Mod) -> #{type => string, example => <<"127.0.0.1:80">>}; +typename_to_spec("ip_ports()", _Mod) -> #{type => string, example => <<"127.0.0.1:80, 127.0.0.2:80">>}; typename_to_spec("url()", _Mod) -> #{type => string, example => <<"http://127.0.0.1">>}; typename_to_spec("server()", Mod) -> typename_to_spec("ip_port()", Mod); +typename_to_spec("servers()", Mod) -> typename_to_spec("ip_ports()", Mod); typename_to_spec("connect_timeout()", Mod) -> typename_to_spec("timeout()", Mod); typename_to_spec("timeout()", _Mod) -> #{<<"oneOf">> => [#{type => string, example => infinity}, #{type => integer, example => 100}], example => infinity}; From 8cb9482542c80185877e1cd96b07fef7d015e9b6 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 29 Dec 2021 16:57:58 +0800 Subject: [PATCH 41/99] fix(authz): `servers` field use string --- apps/emqx_authz/src/emqx_authz_api_schema.erl | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index b9e6b2def..fb643e9b7 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -182,8 +182,7 @@ definitions() -> mongo_type => #{type => string, enum => [<<"rs">>], example => <<"rs">>}, - servers => #{type => array, - items => #{type => string,example => <<"127.0.0.1:27017">>}}, + servers => #{type => string, example => <<"127.0.0.1:27017, 127.0.0.2:27017">>}, replica_set_name => #{type => string}, pool_size => #{type => integer}, username => #{type => string}, @@ -240,8 +239,7 @@ definitions() -> mongo_type => #{type => string, enum => [<<"sharded">>], example => <<"sharded">>}, - servers => #{type => array, - items => #{type => string,example => <<"127.0.0.1:27017">>}}, + servers => #{type => string,example => <<"127.0.0.1:27017, 127.0.0.2:27017">>}, pool_size => #{type => integer}, username => #{type => string}, password => #{type => string}, @@ -401,8 +399,7 @@ definitions() -> type => string, example => <<"HGETALL mqtt_authz">> }, - servers => #{type => array, - items => #{type => string,example => <<"127.0.0.1:3306">>}}, + servers => #{type => string, example => <<"127.0.0.1:6379, 127.0.0.2:6379">>}, redis_type => #{type => string, enum => [<<"sentinel">>], example => <<"sentinel">>}, @@ -438,8 +435,7 @@ definitions() -> type => string, example => <<"HGETALL mqtt_authz">> }, - servers => #{type => array, - items => #{type => string, example => <<"127.0.0.1:3306">>}}, + servers => #{type => string, example => <<"127.0.0.1:6379, 127.0.0.2:6379">>}, redis_type => #{type => string, enum => [<<"cluster">>], example => <<"cluster">>}, From da68dfc8f47bd863c1c77b50cca665e2287054c1 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 29 Dec 2021 16:58:33 +0800 Subject: [PATCH 42/99] test(authz): `servers` field use string --- apps/emqx_authz/test/emqx_authz_SUITE.erl | 1 + apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl | 10 +++------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index 7038b59e0..e3ec1a8ba 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -102,6 +102,7 @@ set_special_configs(_App) -> <<"query">> => <<"abcb">> }). -define(SOURCE5, #{<<"type">> => <<"redis">>, + <<"redis_type">> => <<"single">>, <<"enable">> => true, <<"server">> => <<"127.0.0.1:27017">>, <<"pool_size">> => 1, diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index b1b6676ad..a448fa25f 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -37,9 +37,7 @@ -define(SOURCE2, #{<<"type">> => <<"mongodb">>, <<"enable">> => true, <<"mongo_type">> => <<"sharded">>, - <<"servers">> => [<<"127.0.0.1:27017">>, - <<"192.168.0.1:27017">> - ], + <<"servers">> => <<"127.0.0.1:27017, 192.168.0.1:27017">>, <<"pool_size">> => 1, <<"database">> => <<"mqtt">>, <<"ssl">> => #{<<"enable">> => false}, @@ -70,9 +68,7 @@ }). -define(SOURCE5, #{<<"type">> => <<"redis">>, <<"enable">> => true, - <<"servers">> => [<<"127.0.0.1:6379">>, - <<"127.0.0.1:6380">> - ], + <<"servers">> => <<"127.0.0.1:6379, 127.0.0.1:6380">>, <<"pool_size">> => 1, <<"database">> => 0, <<"password">> => <<"ee">>, @@ -88,7 +84,7 @@ }). all() -> - []. %% Todo: Waiting for @terry-xiaoyu to fix the config_not_found error + []. %% Todo: Waiting for fixing the ssl cert test. % emqx_common_test_helpers:all(?MODULE). groups() -> From 0e62a6772c177f0f54b472dd3eeb73fc7a5b941e Mon Sep 17 00:00:00 2001 From: lafirest Date: Thu, 30 Dec 2021 16:58:33 +0800 Subject: [PATCH 43/99] fix(emqx_slow_subs): fix config update error --- apps/emqx_slow_subs/src/emqx_slow_subs.erl | 14 ++++++++---- .../emqx_slow_subs/src/emqx_slow_subs_api.erl | 3 +-- .../test/emqx_slow_subs_api_SUITE.erl | 22 +++++++++++++++++++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.erl b/apps/emqx_slow_subs/src/emqx_slow_subs.erl index 445465e1f..10cb2a4e7 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs.erl @@ -23,7 +23,7 @@ -include_lib("emqx_slow_subs/include/emqx_slow_subs.hrl"). -export([ start_link/0, on_stats_update/2, update_settings/1 - , clear_history/0, init_topk_tab/0 + , clear_history/0, init_topk_tab/0, post_config_update/5 ]). %% gen_server callbacks @@ -123,8 +123,8 @@ on_stats_update(#{clientid := ClientId, clear_history() -> gen_server:call(?MODULE, ?FUNCTION_NAME, ?DEF_CALL_TIMEOUT). -update_settings(Enable) -> - gen_server:call(?MODULE, {?FUNCTION_NAME, Enable}, ?DEF_CALL_TIMEOUT). +update_settings(Conf) -> + emqx_conf:update([emqx_slow_subs], Conf, #{override_to => cluster}). init_topk_tab() -> case ets:whereis(?TOPK_TAB) of @@ -138,11 +138,16 @@ init_topk_tab() -> ?TOPK_TAB end. +post_config_update(_KeyPath, _UpdateReq, NewConf, _OldConf, _AppEnvs) -> + gen_server:call(?MODULE, {update_settings, NewConf}, ?DEF_CALL_TIMEOUT). + %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- init([]) -> + emqx_conf:add_handler([emqx_slow_subs], ?MODULE), + InitState = #{enable => false, last_tick_at => 0, expire_timer => undefined, @@ -152,7 +157,8 @@ init([]) -> Enable = emqx:get_config([emqx_slow_subs, enable]), {ok, check_enable(Enable, InitState)}. -handle_call({update_settings, Enable}, _From, State) -> +handle_call({update_settings, #{enable := Enable} = Conf}, _From, State) -> + emqx_config:put([emqx_slow_subs], Conf), State2 = check_enable(Enable, State), {reply, ok, State2}; diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl index 4c5873ab9..fb102d80c 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl @@ -107,6 +107,5 @@ settings(get, _) -> {200, emqx:get_raw_config([?APP_NAME], #{})}; settings(put, #{body := Body}) -> - {ok, #{config := #{enable := Enable}}} = emqx:update_config([?APP], Body), - _ = emqx_slow_subs:update_settings(Enable), + _ = emqx_slow_subs:update_settings(Body), {200, emqx:get_raw_config([?APP_NAME], #{})}. diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl index 009feda01..01bfd7f26 100644 --- a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl +++ b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl @@ -32,6 +32,7 @@ -define(BASE_PATH, "api"). -define(NOW, erlang:system_time(millisecond)). +-define(CLUSTER_RPC_SHARD, emqx_cluster_rpc_shard). -define(CONF_DEFAULT, <<""" emqx_slow_subs @@ -49,23 +50,42 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + application:load(emqx_conf), + ok = ekka:start(), + ok = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], infinity), + meck:new(emqx_alarm, [non_strict, passthrough, no_link]), + meck:expect(emqx_alarm, activate, 3, ok), + meck:expect(emqx_alarm, deactivate, 3, ok), + ok = emqx_config:init_load(emqx_slow_subs_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_slow_subs]), {ok, _} = application:ensure_all_started(emqx_authn), Config. end_per_suite(Config) -> + ekka:stop(), + mria:stop(), + mria_mnesia:delete_schema(), + meck:unload(emqx_alarm), + application:stop(emqx_authn), emqx_mgmt_api_test_util:end_suite([emqx_slow_subs]), Config. init_per_testcase(_, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(), application:ensure_all_started(emqx_slow_subs), timer:sleep(500), Config. end_per_testcase(_, Config) -> application:stop(emqx_slow_subs), + case erlang:whereis(node()) of + undefined -> ok; + P -> + erlang:unlink(P), + erlang:exit(P, kill) + end, Config. t_get_history(_) -> @@ -119,6 +139,8 @@ t_settting(_) -> auth_header_() ), + timer:sleep(1000), + GetReturn = decode_json(GetData), ?assertEqual(Conf2, GetReturn), From c2e36b041b26b8b3c5c7ab6b1eb4813240c18e69 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 30 Dec 2021 18:53:09 +0800 Subject: [PATCH 44/99] fix: dashboard return type --- apps/emqx_dashboard/src/emqx_dashboard.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index e79b0210c..07d959b8e 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -151,9 +151,9 @@ authorize(Req) -> ok -> ok; {error, token_timeout} -> - {401, <<"TOKEN_TIME_OUT">>, <<"Token expired, get new token by POST /login">>}; + {401, 'TOKEN_TIME_OUT', <<"Token expired, get new token by POST /login">>}; {error, not_found} -> - {401, <<"BAD_TOKEN">>, <<"Get a token by POST /login">>} + {401, 'BAD_TOKEN', <<"Get a token by POST /login">>} end; _ -> return_unauthorized(<<"AUTHORIZATION_HEADER_ERROR">>, From 489fb7f806b69ff9b59f8310c5da2e8fa93936aa Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 29 Dec 2021 22:54:38 +0800 Subject: [PATCH 45/99] fix(trace): copy binary:encode_hex/2 from binary.erl --- apps/emqx/include/logger.hrl | 15 +++- apps/emqx/src/emqx_broker.erl | 4 +- apps/emqx/src/emqx_cm.erl | 27 +++--- apps/emqx/src/emqx_connection.erl | 2 +- apps/emqx/src/emqx_flapping.erl | 4 +- apps/emqx/src/emqx_logger.erl | 10 +-- apps/emqx/src/emqx_logger_textfmt.erl | 73 +++++++++++++-- apps/emqx/src/emqx_packet.erl | 88 +++++++++++++++++-- apps/emqx/src/emqx_schema.erl | 10 +-- apps/emqx/src/emqx_trace/emqx_trace.erl | 47 +++++----- .../src/emqx_trace/emqx_trace_formatter.erl | 21 ++--- .../src/emqx_trace/emqx_trace_handler.erl | 11 +-- apps/emqx/test/emqx_trace_handler_SUITE.erl | 19 ++-- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 6 +- apps/emqx_conf/src/emqx_conf_schema.erl | 11 +-- .../src/emqx_connector_pgsql.erl | 3 +- .../emqx_management/src/emqx_mgmt_api_app.erl | 2 +- .../src/emqx_mgmt_api_trace.erl | 5 +- apps/emqx_management/src/emqx_mgmt_cli.erl | 38 ++++---- 19 files changed, 265 insertions(+), 131 deletions(-) diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index 2398a7dee..d549e2ccb 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -67,13 +67,24 @@ case logger:allow(Level, ?MODULE) of true -> logger:log(Level, (Data), (Meta#{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} - , line => ?LINE + , line => ?LINE })); false -> ok end). --define(TRACE(Event, Msg, Meta), emqx_trace:log(Event, Msg, Meta)). +-define(TRACE_FILTER, emqx_trace_filter). + +%% Only evaluate when necessary +-define(TRACE(Event, Msg, Meta), + begin + case persistent_term:get(?TRACE_FILTER, undefined) of + undefined -> ok; + [] -> ok; + List -> + emqx_trace:log(List, Event, Msg, Meta) + end + end). %% print to 'user' group leader -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)). diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index 4085b6130..9dbfb0b43 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -205,8 +205,8 @@ publish(Msg) when is_record(Msg, message) -> emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'), case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of #message{headers = #{allow_publish := false}, topic = Topic} -> - Message = emqx_message:to_log_map(Msg), - ?TRACE("MQTT", "msg_publish_not_allowed", #{message => Message, topic => Topic}), + ?TRACE("MQTT", "msg_publish_not_allowed", #{message => emqx_message:to_log_map(Msg), + topic => Topic}), []; Msg1 = #message{topic = Topic} -> emqx_persistent_session:persist_message(Msg1), diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index a2101efa1..eae8dd43d 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -375,7 +375,7 @@ discard_session(ClientId) when is_binary(ClientId) -> -spec kick_or_kill(kick | discard, module(), pid()) -> ok. kick_or_kill(Action, ConnMod, Pid) -> try - %% this is essentailly a gen_server:call implemented in emqx_connection + %% this is essentially a gen_server:call implemented in emqx_connection %% and emqx_ws_connection. %% the handle_call is implemented in emqx_channel ok = apply(ConnMod, call, [Pid, Action, ?T_KICK]) @@ -390,19 +390,12 @@ kick_or_kill(Action, ConnMod, Pid) -> ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}); _ : {timeout, {gen_server, call, _}} -> ?tp(warning, "session_kick_timeout", - #{pid => Pid, - action => Action, - stale_channel => stale_channel_info(Pid) - }), + #{pid => Pid, action => Action, stale_channel => stale_channel_info(Pid)}), ok = force_kill(Pid); _ : Error : St -> ?tp(error, "session_kick_exception", - #{pid => Pid, - action => Action, - reason => Error, - stacktrace => St, - stale_channel => stale_channel_info(Pid) - }), + #{pid => Pid, action => Action, reason => Error, stacktrace => St, + stale_channel => stale_channel_info(Pid)}), ok = force_kill(Pid) end. @@ -449,21 +442,21 @@ kick_session(Action, ClientId, ChanPid) -> , error => Error , reason => Reason }, - #{clientid => unicode:characters_to_list(ClientId, utf8)}) + #{clientid => ClientId}) end. kick_session(ClientId) -> case lookup_channels(ClientId) of [] -> ?SLOG(warning, #{msg => "kicked_an_unknown_session"}, - #{clientid => unicode:characters_to_list(ClientId, utf8)}), + #{clientid => ClientId}), ok; ChanPids -> case length(ChanPids) > 1 of true -> ?SLOG(warning, #{msg => "more_than_one_channel_found", chan_pids => ChanPids}, - #{clientid => unicode:characters_to_list(ClientId, utf8)}); + #{clientid => ClientId}); false -> ok end, lists:foreach(fun(Pid) -> kick_session(ClientId, Pid) end, ChanPids) @@ -480,12 +473,12 @@ with_channel(ClientId, Fun) -> Pids -> Fun(lists:last(Pids)) end. -%% @doc Get all registed channel pids. Debugg/test interface +%% @doc Get all registered channel pids. Debug/test interface all_channels() -> Pat = [{{'_', '$1'}, [], ['$1']}], ets:select(?CHAN_TAB, Pat). -%% @doc Get all registed clientIDs. Debugg/test interface +%% @doc Get all registered clientIDs. Debug/test interface all_client_ids() -> Pat = [{{'$1', '_'}, [], ['$1']}], ets:select(?CHAN_TAB, Pat). @@ -513,7 +506,7 @@ lookup_channels(local, ClientId) -> rpc_call(Node, Fun, Args, Timeout) -> case rpc:call(Node, ?MODULE, Fun, Args, 2 * Timeout) of {badrpc, Reason} -> - %% since eqmx app 4.3.10, the 'kick' and 'discard' calls hanndler + %% since emqx app 4.3.10, the 'kick' and 'discard' calls handler %% should catch all exceptions and always return 'ok'. %% This leaves 'badrpc' only possible when there is problem %% calling the remote node. diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index e1dab3260..d334ac23e 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -754,7 +754,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) -> <<>> -> ?SLOG(warning, #{ msg => "packet_is_discarded", reason => "frame_is_too_large", - packet => emqx_packet:format(Packet, null) + packet => emqx_packet:format(Packet, hidden) }), ok = emqx_metrics:inc('delivery.dropped.too_large'), ok = emqx_metrics:inc('delivery.dropped'), diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index df7c7523f..b34819e53 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -121,7 +121,7 @@ handle_cast({detected, #flapping{clientid = ClientId, peer_host => fmt_host(PeerHost), detect_cnt => DetectCnt, wind_time_in_ms => WindTime - }, #{clientid => unicode:characters_to_list(ClientId, utf8)}), + }, #{clientid => ClientId}), Now = erlang:system_time(second), Banned = #banned{who = {clientid, ClientId}, by = <<"flapping detector">>, @@ -136,7 +136,7 @@ handle_cast({detected, #flapping{clientid = ClientId, peer_host => fmt_host(PeerHost), detect_cnt => DetectCnt, interval => Interval - }, #{clientid => unicode:characters_to_list(ClientId, utf8)}) + }, #{clientid => ClientId}) end, {noreply, State}; diff --git a/apps/emqx/src/emqx_logger.erl b/apps/emqx/src/emqx_logger.erl index 79ac5e6b8..66274a711 100644 --- a/apps/emqx/src/emqx_logger.erl +++ b/apps/emqx/src/emqx_logger.erl @@ -197,15 +197,7 @@ critical(Metadata, Format, Args) when is_map(Metadata) -> set_metadata_clientid(<<>>) -> ok; set_metadata_clientid(ClientId) -> - try - %% try put string format client-id metadata so - %% so the log is not like <<"...">> - Id = unicode:characters_to_list(ClientId, utf8), - set_proc_metadata(#{clientid => Id}) - catch - _: _-> - ok - end. + set_proc_metadata(#{clientid => ClientId}). -spec(set_metadata_peername(peername_str()) -> ok). set_metadata_peername(Peername) -> diff --git a/apps/emqx/src/emqx_logger_textfmt.erl b/apps/emqx/src/emqx_logger_textfmt.erl index 986c0fd8a..4e7dbcf14 100644 --- a/apps/emqx/src/emqx_logger_textfmt.erl +++ b/apps/emqx/src/emqx_logger_textfmt.erl @@ -18,22 +18,77 @@ -export([format/2]). -export([check_config/1]). +-export([try_format_unicode/1]). check_config(X) -> logger_formatter:check_config(X). -format(#{msg := {report, Report}, meta := Meta} = Event, Config) when is_map(Report) -> - logger_formatter:format(Event#{msg := {report, enrich(Report, Meta)}}, Config); -format(#{msg := Msg, meta := Meta} = Event, Config) -> - NewMsg = enrich_fmt(Msg, Meta), - logger_formatter:format(Event#{msg := NewMsg}, Config). +format(#{msg := {report, Report0}, meta := Meta} = Event, Config) when is_map(Report0) -> + Report1 = enrich_report_mfa(Report0, Meta), + Report2 = enrich_report_clientid(Report1, Meta), + Report3 = enrich_report_peername(Report2, Meta), + Report4 = enrich_report_topic(Report3, Meta), + logger_formatter:format(Event#{msg := {report, Report4}}, Config); +format(#{msg := {string, String}} = Event, Config) -> + format(Event#{msg => {"~ts ", String}}, Config); +format(#{msg := Msg0, meta := Meta} = Event, Config) -> + Msg1 = enrich_client_info(Msg0, Meta), + Msg2 = enrich_mfa(Msg1, Meta), + Msg3 = enrich_topic(Msg2, Meta), + logger_formatter:format(Event#{msg := Msg3}, Config). -enrich(Report, #{mfa := Mfa, line := Line}) -> +try_format_unicode(Char) -> + List = + try + case unicode:characters_to_list(Char) of + {error, _, _} -> error; + {incomplete, _, _} -> error; + Binary -> Binary + end + catch _:_ -> + error + end, + case List of + error -> io_lib:format("~0p", [Char]); + _ -> List + end. + +enrich_report_mfa(Report, #{mfa := Mfa, line := Line}) -> Report#{mfa => mfa(Mfa), line => Line}; -enrich(Report, _) -> Report. +enrich_report_mfa(Report, _) -> Report. -enrich_fmt({Fmt, Args}, #{mfa := Mfa, line := Line}) when is_list(Fmt) -> +enrich_report_clientid(Report, #{clientid := ClientId}) -> + Report#{clientid => try_format_unicode(ClientId)}; +enrich_report_clientid(Report, _) -> Report. + +enrich_report_peername(Report, #{peername := Peername}) -> + Report#{peername => Peername}; +enrich_report_peername(Report, _) -> Report. + +%% clientid and peername always in emqx_conn's process metadata. +%% topic can be put in meta using ?SLOG/3, or put in msg's report by ?SLOG/2 +enrich_report_topic(Report, #{topic := Topic}) -> + Report#{topic => try_format_unicode(Topic)}; +enrich_report_topic(Report = #{topic := Topic}, _) -> + Report#{topic => try_format_unicode(Topic)}; +enrich_report_topic(Report, _) -> Report. + +enrich_mfa({Fmt, Args}, #{mfa := Mfa, line := Line}) when is_list(Fmt) -> {Fmt ++ " mfa: ~ts line: ~w", Args ++ [mfa(Mfa), Line]}; -enrich_fmt(Msg, _) -> +enrich_mfa(Msg, _) -> + Msg. + +enrich_client_info({Fmt, Args}, #{clientid := ClientId, peername := Peer}) when is_list(Fmt) -> + {" ~ts@~ts " ++ Fmt, [ClientId, Peer | Args] }; +enrich_client_info({Fmt, Args}, #{clientid := ClientId}) when is_list(Fmt) -> + {" ~ts " ++ Fmt, [ClientId | Args]}; +enrich_client_info({Fmt, Args}, #{peername := Peer}) when is_list(Fmt) -> + {" ~ts " ++ Fmt, [Peer | Args]}; +enrich_client_info(Msg, _) -> + Msg. + +enrich_topic({Fmt, Args}, #{topic := Topic}) when is_list(Fmt) -> + {" topic: ~ts" ++ Fmt, [Topic | Args]}; +enrich_topic(Msg, _) -> Msg. mfa({M, F, A}) -> atom_to_list(M) ++ ":" ++ atom_to_list(F) ++ "/" ++ integer_to_list(A). diff --git a/apps/emqx/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl index 53e4eabfe..23b8390e5 100644 --- a/apps/emqx/src/emqx_packet.erl +++ b/apps/emqx/src/emqx_packet.erl @@ -48,6 +48,8 @@ , format/2 ]). +-export([encode_hex/1]). + -define(TYPE_NAMES, { 'CONNECT' , 'CONNACK' @@ -440,7 +442,7 @@ will_msg(#mqtt_packet_connect{clientid = ClientId, format(Packet) -> format(Packet, emqx_trace_handler:payload_encode()). %% @doc Format packet --spec(format(emqx_types:packet(), hex | text | null) -> iolist()). +-spec(format(emqx_types:packet(), hex | text | hidden) -> iolist()). format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}, PayloadEncode) -> HeaderIO = format_header(Header), case format_variable(Variable, Payload, PayloadEncode) of @@ -504,11 +506,13 @@ format_variable(#mqtt_packet_puback{packet_id = PacketId, format_variable(#mqtt_packet_subscribe{packet_id = PacketId, topic_filters = TopicFilters}, _) -> - io_lib:format("PacketId=~p, TopicFilters=~0p", [PacketId, TopicFilters]); + [io_lib:format("PacketId=~p ", [PacketId]), "TopicFilters=", + format_topic_filters(TopicFilters)]; format_variable(#mqtt_packet_unsubscribe{packet_id = PacketId, topic_filters = Topics}, _) -> - io_lib:format("PacketId=~p, TopicFilters=~0p", [PacketId, Topics]); + [io_lib:format("PacketId=~p ", [PacketId]), "TopicFilters=", + format_topic_filters(Topics)]; format_variable(#mqtt_packet_suback{packet_id = PacketId, reason_codes = ReasonCodes}, _) -> @@ -527,9 +531,83 @@ format_password(undefined) -> "undefined"; format_password(_Password) -> "******". format_payload(Payload, text) -> ["Payload=", io_lib:format("~ts", [Payload])]; -format_payload(Payload, hex) -> ["Payload(hex)=", binary:encode_hex(Payload)]; -format_payload(_, null) -> "Payload=******". +format_payload(Payload, hex) -> ["Payload(hex)=", encode_hex(Payload)]; +format_payload(_, hidden) -> "Payload=******". i(true) -> 1; i(false) -> 0; i(I) when is_integer(I) -> I. + +format_topic_filters(Filters) -> + ["[", + lists:join(",", + lists:map( + fun({TopicFilter, SubOpts}) -> + io_lib:format("~ts(~p)", [TopicFilter, SubOpts]); + (TopicFilter) -> + io_lib:format("~ts", [TopicFilter]) + end, Filters)), + "]"]. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Hex encoding functions +%% Copy from binary:encode_hex/1 (was only introduced in OTP24). +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +-define(HEX(X), (hex(X)):16). +-compile({inline,[hex/1]}). +-spec encode_hex(Bin) -> Bin2 when + Bin :: binary(), + Bin2 :: <<_:_*16>>. +encode_hex(Data) when byte_size(Data) rem 8 =:= 0 -> + << <> || <> <= Data >>; +encode_hex(Data) when byte_size(Data) rem 7 =:= 0 -> + << <> || <> <= Data >>; +encode_hex(Data) when byte_size(Data) rem 6 =:= 0 -> + << <> || <> <= Data >>; +encode_hex(Data) when byte_size(Data) rem 5 =:= 0 -> + << <> || <> <= Data >>; +encode_hex(Data) when byte_size(Data) rem 4 =:= 0 -> + << <> || <> <= Data >>; +encode_hex(Data) when byte_size(Data) rem 3 =:= 0 -> + << <> || <> <= Data >>; +encode_hex(Data) when byte_size(Data) rem 2 =:= 0 -> + << <> || <> <= Data >>; +encode_hex(Data) when is_binary(Data) -> + << <> || <> <= Data >>; +encode_hex(Bin) -> + erlang:error(badarg, [Bin]). + +hex(X) -> + element( + X+1, {16#3030, 16#3031, 16#3032, 16#3033, 16#3034, 16#3035, 16#3036, 16#3037, 16#3038, 16#3039, 16#3041, + 16#3042, 16#3043, 16#3044, 16#3045, 16#3046, + 16#3130, 16#3131, 16#3132, 16#3133, 16#3134, 16#3135, 16#3136, 16#3137, 16#3138, 16#3139, 16#3141, + 16#3142, 16#3143, 16#3144, 16#3145, 16#3146, + 16#3230, 16#3231, 16#3232, 16#3233, 16#3234, 16#3235, 16#3236, 16#3237, 16#3238, 16#3239, 16#3241, + 16#3242, 16#3243, 16#3244, 16#3245, 16#3246, + 16#3330, 16#3331, 16#3332, 16#3333, 16#3334, 16#3335, 16#3336, 16#3337, 16#3338, 16#3339, 16#3341, + 16#3342, 16#3343, 16#3344, 16#3345, 16#3346, + 16#3430, 16#3431, 16#3432, 16#3433, 16#3434, 16#3435, 16#3436, 16#3437, 16#3438, 16#3439, 16#3441, + 16#3442, 16#3443, 16#3444, 16#3445, 16#3446, + 16#3530, 16#3531, 16#3532, 16#3533, 16#3534, 16#3535, 16#3536, 16#3537, 16#3538, 16#3539, 16#3541, + 16#3542, 16#3543, 16#3544, 16#3545, 16#3546, + 16#3630, 16#3631, 16#3632, 16#3633, 16#3634, 16#3635, 16#3636, 16#3637, 16#3638, 16#3639, 16#3641, + 16#3642, 16#3643, 16#3644, 16#3645, 16#3646, + 16#3730, 16#3731, 16#3732, 16#3733, 16#3734, 16#3735, 16#3736, 16#3737, 16#3738, 16#3739, 16#3741, + 16#3742, 16#3743, 16#3744, 16#3745, 16#3746, + 16#3830, 16#3831, 16#3832, 16#3833, 16#3834, 16#3835, 16#3836, 16#3837, 16#3838, 16#3839, 16#3841, + 16#3842, 16#3843, 16#3844, 16#3845, 16#3846, + 16#3930, 16#3931, 16#3932, 16#3933, 16#3934, 16#3935, 16#3936, 16#3937, 16#3938, 16#3939, 16#3941, + 16#3942, 16#3943, 16#3944, 16#3945, 16#3946, + 16#4130, 16#4131, 16#4132, 16#4133, 16#4134, 16#4135, 16#4136, 16#4137, 16#4138, 16#4139, 16#4141, + 16#4142, 16#4143, 16#4144, 16#4145, 16#4146, + 16#4230, 16#4231, 16#4232, 16#4233, 16#4234, 16#4235, 16#4236, 16#4237, 16#4238, 16#4239, 16#4241, + 16#4242, 16#4243, 16#4244, 16#4245, 16#4246, + 16#4330, 16#4331, 16#4332, 16#4333, 16#4334, 16#4335, 16#4336, 16#4337, 16#4338, 16#4339, 16#4341, + 16#4342, 16#4343, 16#4344, 16#4345, 16#4346, + 16#4430, 16#4431, 16#4432, 16#4433, 16#4434, 16#4435, 16#4436, 16#4437, 16#4438, 16#4439, 16#4441, + 16#4442, 16#4443, 16#4444, 16#4445, 16#4446, + 16#4530, 16#4531, 16#4532, 16#4533, 16#4534, 16#4535, 16#4536, 16#4537, 16#4538, 16#4539, 16#4541, + 16#4542, 16#4543, 16#4544, 16#4545, 16#4546, + 16#4630, 16#4631, 16#4632, 16#4633, 16#4634, 16#4635, 16#4636, 16#4637, 16#4638, 16#4639, 16#4641, + 16#4642, 16#4643, 16#4644, 16#4645, 16#4646}). diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 123ee5379..fc90ba1bb 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -985,13 +985,13 @@ fields("latency_stats") -> desc => "the number of smaples for calculate the average latency of delivery"})} ]; fields("trace") -> - [ {"payload_encode", sc(hoconsc:enum([hex, text, null]), #{ + [ {"payload_encode", sc(hoconsc:enum([hex, text, hidden]), #{ default => text, desc => """ -Determine the format of the payload format in the trace file.
-- `text`: Text-based protocol or plain text protocol. It is recommended when payload is json encode.
-- `hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.
-- `null`: Don't show payload in trace log file. +Determine the format of the payload format in the trace file.
+`text`: Text-based protocol or plain text protocol. It is recommended when payload is json encode.
+`hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.
+`hidden`: payload is obfuscated as `******` """ })} ]. diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 3869d300d..5af0d156e 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -26,7 +26,7 @@ -export([ publish/1 , subscribe/3 , unsubscribe/2 - , log/3 + , log/4 ]). -export([ start_link/0 @@ -52,8 +52,7 @@ -define(TRACE, ?MODULE). -define(MAX_SIZE, 30). --define(TRACE_FILTER, emqx_trace_filter). --define(OWN_KEYS,[level,filters,filter_default,handlers]). +-define(OWN_KEYS, [level, filters, filter_default, handlers]). -ifdef(TEST). -export([ log_file/2 @@ -94,18 +93,14 @@ unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore; unsubscribe(Topic, SubOpts) -> ?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}). -log(Event, Msg, Meta0) -> - case persistent_term:get(?TRACE_FILTER, undefined) of - undefined -> ok; - List -> - Meta = - case logger:get_process_metadata() of - undefined -> Meta0; - ProcMeta -> maps:merge(ProcMeta, Meta0) - end, - Log = #{level => trace, event => Event, meta => Meta, msg => Msg}, - log_filter(List, Log) - end. +log(List, Event, Msg, Meta0) -> + Meta = + case logger:get_process_metadata() of + undefined -> Meta0; + ProcMeta -> maps:merge(ProcMeta, Meta0) + end, + Log = #{level => trace, event => Event, meta => Meta, msg => Msg}, + log_filter(List, Log). log_filter([], _Log) -> ok; log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) -> @@ -196,7 +191,7 @@ update(Name, Enable) -> transaction(Tran). check() -> - erlang:send(?MODULE, {mnesia_table_event, check}). + gen_server:call(?MODULE, check). -spec get_trace_filename(Name :: binary()) -> {ok, FileName :: string()} | {error, not_found}. @@ -241,6 +236,9 @@ init([]) -> update_trace_handler(), {ok, #{timer => TRef, monitors => #{}}}. +handle_call(check, _From, State) -> + {_, NewState} = handle_info({mnesia_table_event, check}, State), + {reply, ok, NewState}; handle_call(Req, _From, State) -> ?SLOG(error, #{unexpected_call => Req}), {reply, ok, State}. @@ -259,8 +257,7 @@ handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitor lists:foreach(fun file:delete/1, Files), {noreply, State#{monitors => NewMonitors}} end; -handle_info({timeout, TRef, update_trace}, - #{timer := TRef} = State) -> +handle_info({timeout, TRef, update_trace}, #{timer := TRef} = State) -> Traces = get_enable_trace(), NextTRef = update_trace(Traces), update_trace_handler(), @@ -344,10 +341,10 @@ disable_finished(Traces) -> start_trace(Traces, Started0) -> Started = lists:map(fun(#{name := Name}) -> Name end, Started0), - lists:foldl(fun(#?TRACE{name = Name} = Trace, {Running, StartedAcc}) -> + lists:foldl(fun(#?TRACE{name = Name} = Trace, + {Running, StartedAcc}) -> case lists:member(Name, StartedAcc) of - true -> - {[Name | Running], StartedAcc}; + true -> {[Name | Running], StartedAcc}; false -> case start_trace(Trace) of ok -> {[Name | Running], [Name | StartedAcc]}; @@ -366,9 +363,11 @@ start_trace(Trace) -> emqx_trace_handler:install(Who, debug, log_file(Name, Start)). stop_trace(Finished, Started) -> - lists:foreach(fun(#{name := Name, type := Type}) -> + lists:foreach(fun(#{name := Name, type := Type, filter := Filter}) -> case lists:member(Name, Finished) of - true -> emqx_trace_handler:uninstall(Type, Name); + true -> + ?TRACE("API", "trace_stopping", #{Type => Filter}), + emqx_trace_handler:uninstall(Type, Name); false -> ok end end, Started). @@ -455,7 +454,7 @@ to_trace(#{type := ip_address, ip_address := Filter} = Trace, Rec) -> case validate_ip_address(Filter) of ok -> Trace0 = maps:without([type, ip_address], Trace), - to_trace(Trace0, Rec#?TRACE{type = ip_address, filter = Filter}); + to_trace(Trace0, Rec#?TRACE{type = ip_address, filter = binary_to_list(Filter)}); Error -> Error end; to_trace(#{type := Type}, _Rec) -> {error, io_lib:format("required ~s field", [Type])}; diff --git a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl index a9c4ca31d..2ef142d38 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl @@ -25,7 +25,7 @@ format(#{level := trace, event := Event, meta := Meta, msg := Msg}, #{payload_encode := PEncode}) -> Time = calendar:system_time_to_rfc3339(erlang:system_time(second)), - ClientId = maps:get(clientid, Meta, ""), + ClientId = to_iolist(maps:get(clientid, Meta, "")), Peername = maps:get(peername, Meta, ""), MetaBin = format_meta(Meta, PEncode), [Time, " [", Event, "] ", ClientId, "@", Peername, " msg: ", Msg, MetaBin, "\n"]; @@ -39,10 +39,7 @@ format_meta(Meta0, Encode) -> Meta1 = maps:without([msg, clientid, peername, packet, payload], Meta0), case Meta1 =:= #{} of true -> [Packet, Payload]; - false -> - Meta2 = lists:map(fun({K, V}) -> [to_iolist(K), ": ", to_iolist(V)] end, - maps:to_list(Meta1)), - [Packet, ", ", lists:join(",", Meta2), Payload] + false -> [Packet, ", ", map_to_iolist(Meta1), Payload] end. format_packet(undefined, _) -> ""; @@ -50,12 +47,16 @@ format_packet(Packet, Encode) -> [", packet: ", emqx_packet:format(Packet, Encod format_payload(undefined, _) -> ""; format_payload(Payload, text) -> [", payload: ", io_lib:format("~ts", [Payload])]; -format_payload(Payload, hex) -> [", payload(hex): ", binary:encode_hex(Payload)]; -format_payload(_, null) -> ", payload=******". +format_payload(Payload, hex) -> [", payload(hex): ", emqx_packet:encode_hex(Payload)]; +format_payload(_, hidden) -> ", payload=******". to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom); to_iolist(Int) when is_integer(Int) -> integer_to_list(Int); to_iolist(Float) when is_float(Float) -> float_to_list(Float, [{decimals, 2}]); -to_iolist(Bin)when is_binary(Bin) -> unicode:characters_to_binary(Bin); -to_iolist(List) when is_list(List) -> unicode:characters_to_list(List); -to_iolist(Term) -> io_lib:format("~0p", [Term]). +to_iolist(SubMap) when is_map(SubMap) -> ["[", map_to_iolist(SubMap), "]"]; +to_iolist(Char) -> emqx_logger_textfmt:try_format_unicode(Char). + +map_to_iolist(Map) -> + lists:join(",", + lists:map(fun({K, V}) -> [to_iolist(K), ": ", to_iolist(V)] end, + maps:to_list(Map))). diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl index f8a5f3828..320421309 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl @@ -119,11 +119,11 @@ uninstall(HandlerId) -> running() -> lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)). --spec filter_clientid(logger:log_event(), {string(), atom()}) -> logger:log_event() | stop. +-spec filter_clientid(logger:log_event(), {binary(), atom()}) -> logger:log_event() | stop. filter_clientid(#{meta := #{clientid := ClientId}} = Log, {ClientId, _Name}) -> Log; filter_clientid(_Log, _ExpectId) -> stop. --spec filter_topic(logger:log_event(), {string(), atom()}) -> logger:log_event() | stop. +-spec filter_topic(logger:log_event(), {binary(), atom()}) -> logger:log_event() | stop. filter_topic(#{meta := #{topic := Topic}} = Log, {TopicFilter, _Name}) -> case emqx_topic:match(Topic, TopicFilter) of true -> Log; @@ -140,7 +140,7 @@ filter_ip_address(#{meta := #{peername := Peername}} = Log, {IP, _Name}) -> filter_ip_address(_Log, _ExpectId) -> stop. filters(#{type := clientid, filter := Filter, name := Name}) -> - [{clientid, {fun ?MODULE:filter_clientid/2, {ensure_list(Filter), Name}}}]; + [{clientid, {fun ?MODULE:filter_clientid/2, {Filter, Name}}}]; filters(#{type := topic, filter := Filter, name := Name}) -> [{topic, {fun ?MODULE:filter_topic/2, {ensure_bin(Filter), Name}}}]; filters(#{type := ip_address, filter := Filter, name := Name}) -> @@ -149,8 +149,9 @@ filters(#{type := ip_address, filter := Filter, name := Name}) -> formatter(#{type := _Type}) -> {emqx_trace_formatter, #{ - template => [], - single_line => false, + %% template is for ?SLOG message not ?TRACE. + template => [time," [",level,"] ", msg,"\n"], + single_line => true, max_size => unlimited, depth => unlimited, payload_encode => payload_encode() diff --git a/apps/emqx/test/emqx_trace_handler_SUITE.erl b/apps/emqx/test/emqx_trace_handler_SUITE.erl index f3ab7b5b8..1224fdac9 100644 --- a/apps/emqx/test/emqx_trace_handler_SUITE.erl +++ b/apps/emqx/test/emqx_trace_handler_SUITE.erl @@ -32,19 +32,22 @@ all() -> [t_trace_clientid, t_trace_topic, t_trace_ip_address, t_trace_clientid_ init_per_suite(Config) -> emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([emqx_modules]), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([emqx_modules]). + emqx_common_test_helpers:stop_apps([]). init_per_testcase(t_trace_clientid, Config) -> + init(), Config; init_per_testcase(_Case, Config) -> _ = [logger:remove_handler(Id) ||#{id := Id} <- emqx_trace_handler:running()], + init(), Config. end_per_testcase(_Case, _Config) -> + terminate(), ok. t_trace_clientid(_Config) -> @@ -66,11 +69,11 @@ t_trace_clientid(_Config) -> ?assert(filelib:is_regular("tmp/client3.log")), %% Get current traces - ?assertMatch([#{type := clientid, filter := "client", name := <<"CLI-client1">>, + ?assertMatch([#{type := clientid, filter := <<"client">>, name := <<"CLI-client1">>, level := debug, dst := "tmp/client.log"}, - #{type := clientid, filter := "client2", name := <<"CLI-client2">> + #{type := clientid, filter := <<"client2">>, name := <<"CLI-client2">> , level := debug, dst := "tmp/client2.log"}, - #{type := clientid, filter := "client3", name := <<"CLI-client3">>, + #{type := clientid, filter := <<"client3">>, name := <<"CLI-client3">>, level := debug, dst := "tmp/client3.log"} ], emqx_trace_handler:running()), @@ -231,3 +234,9 @@ filesync(Name0, Type, Retry) -> ct:sleep(100), filesync(Name, Type, Retry - 1) end. + +init() -> + emqx_trace:start_link(). + +terminate() -> + catch ok = gen_server:stop(emqx_trace, normal, 5000). diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 820eeb859..31e5e52e1 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -43,6 +43,7 @@ groups() -> []. init_per_testcase(_, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authn_test_lib:delete_authenticators( [?CONF_NS_ATOM], ?GLOBAL), @@ -55,8 +56,9 @@ init_per_testcase(_, Config) -> Config. init_per_suite(Config) -> + _ = application:load(emqx_conf), ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_authn, emqx_dashboard], + [emqx_authn, emqx_dashboard], fun set_special_configs/1), ?AUTHN:delete_chain(?GLOBAL), @@ -65,7 +67,7 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([emqx_authn, emqx_dashboard]), + emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authn]), ok. set_special_configs(emqx_dashboard) -> diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 1ea368826..385953b87 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -792,16 +792,7 @@ do_formatter(json, CharsLimit, SingleLine, TimeOffSet, Depth) -> }}; do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth) -> {emqx_logger_textfmt, - #{template => - [time," [",level,"] ", - {clientid, - [{peername, - [clientid,"@",peername," "], - [clientid, " "]}], - [{peername, - [peername," "], - []}]}, - msg,"\n"], + #{template => [time," [",level,"] ", msg,"\n"], chars_limit => CharsLimit, single_line => SingleLine, time_offset => TimeOffSet, diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 5d9d0e1d9..81435a1c5 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -88,7 +88,8 @@ on_query(InstId, QueryParams, AfterQuery, #{poolname := PoolName} = State) -> {prepared_query, Name, SQL} -> {prepared_query, [Name, SQL, []]}; {prepared_query, Name, SQL, Params} -> {prepared_query, [Name, SQL, Params]} end, - ?TRACE("QUERY", "postgresql_connector_received", #{connector => InstId, command => Command, args => Args, state => State}}), + ?TRACE("QUERY", "postgresql_connector_received", + #{connector => InstId, command => Command, args => Args, state => State}), case Result = ecpool:pick_and_do(PoolName, {?MODULE, Command, Args}, no_handover) of {error, Reason} -> ?SLOG(error, #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_app.erl b/apps/emqx_management/src/emqx_mgmt_api_app.erl index dfce3cf30..489d679be 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_app.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_app.erl @@ -149,7 +149,7 @@ api_key(post, #{body := App}) -> Desc = unicode:characters_to_binary(Desc0, unicode), case emqx_mgmt_auth:create(Name, Enable, ExpiredAt, Desc) of {ok, NewApp} -> {200, format(NewApp)}; - {error, Reason} -> {400, Reason} + {error, Reason} -> {400, io_lib:format("~p", [Reason])} end. api_key_by_name(get, #{bindings := #{name := Name}}) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 15277fa8e..296cecea2 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -129,11 +129,10 @@ schema("/trace/:name/log") -> hoconsc:ref(position), hoconsc:ref(node) ], - %% todo response data responses => #{ 200 => [ - {items, hoconsc:mk(binary(), #{example => "BinBinBin"})} + {items, hoconsc:mk(binary(), #{example => "TEXT-LOG-ITEMS"})} | fields(bytes) ++ fields(position) ] } @@ -307,7 +306,7 @@ download_trace_log(get, #{bindings := #{name := Name}}) -> {ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]), emqx_trace:delete_files_after_send(ZipFileName, Zips), Headers = #{ - <<"content-type">> => <<"application/octet-stream">>, + <<"content-type">> => <<"application/x-zip">>, <<"content-disposition">> => iolist_to_binary("attachment; filename=" ++ filename:basename(ZipFile)) }, diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 0084855ab..3a723f33c 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -18,6 +18,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). -include("emqx_mgmt.hrl"). @@ -386,19 +387,19 @@ trace(["list"]) -> emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Type, Filter, Level, Dst]) end, emqx_trace_handler:running()); -trace(["stop", Operation, ClientId]) -> - case trace_type(Operation) of - {ok, Type} -> trace_off(Type, ClientId); +trace(["stop", Operation, Filter0]) -> + case trace_type(Operation, Filter0) of + {ok, Type, Filter} -> trace_off(Type, Filter); error -> trace([]) end; trace(["start", Operation, ClientId, LogFile]) -> trace(["start", Operation, ClientId, LogFile, "all"]); -trace(["start", Operation, Filter, LogFile, Level]) -> - case trace_type(Operation) of - {ok, Type} -> - trace_on(name(Filter), Type, Filter, +trace(["start", Operation, Filter0, LogFile, Level]) -> + case trace_type(Operation, Filter0) of + {ok, Type, Filter} -> + trace_on(name(Filter0), Type, Filter, list_to_existing_atom(Level), LogFile); error -> trace([]) end; @@ -428,13 +429,14 @@ trace_on(Name, Type, Filter, Level, LogFile) -> emqx_ctl:print("[error] trace ~s ~s: ~p~n", [Filter, Name, Error]) end. -trace_off(Who, Filter) -> - case emqx_trace_handler:uninstall(Who, name(Filter)) of +trace_off(Type, Filter) -> + ?TRACE("CLI", "trace_stopping", #{Type => Filter}), + case emqx_trace_handler:uninstall(Type, name(Filter)) of ok -> emqx_trace:check(), - emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Filter]); + emqx_ctl:print("stop tracing ~s ~s successfully~n", [Type, Filter]); {error, Error} -> - emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Who, Filter, Error]) + emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Type, Filter, Error]) end. %%-------------------------------------------------------------------- @@ -463,9 +465,9 @@ traces(["delete", Name]) -> traces(["start", Name, Operation, Filter]) -> traces(["start", Name, Operation, Filter, "900"]); -traces(["start", Name, Operation, Filter, DurationS]) -> - case trace_type(Operation) of - {ok, Type} -> trace_cluster_on(Name, Type, Filter, DurationS); +traces(["start", Name, Operation, Filter0, DurationS]) -> + case trace_type(Operation, Filter0) of + {ok, Type, Filter} -> trace_cluster_on(Name, Type, Filter, DurationS); error -> traces([]) end; @@ -507,10 +509,10 @@ trace_cluster_off(Name) -> {error, Error} -> emqx_ctl:print("[error] Stop cluster_trace ~s: ~p~n", [Name, Error]) end. -trace_type("client") -> {ok, clientid}; -trace_type("topic") -> {ok, topic}; -trace_type("ip_address") -> {ok, ip_address}; -trace_type(_) -> error. +trace_type("client", ClientId) -> {ok, clientid, list_to_binary(ClientId)}; +trace_type("topic", Topic) -> {ok, topic, list_to_binary(Topic)}; +trace_type("ip_address", IP) -> {ok, ip_address, IP}; +trace_type(_, _) -> error. %%-------------------------------------------------------------------- %% @doc Listeners Command From d627d9e69e89aba9e61b6c9b9e512e551316635c Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 24 Dec 2021 11:29:44 +0800 Subject: [PATCH 46/99] chore(dashboard): update dashboard version to v0.14.0 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 49547a03b..e034b5663 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3 export EMQX_DEFAULT_RUNNER = alpine:3.14 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v0.10.0 +export EMQX_DASHBOARD_VERSION ?= v0.14.0 export DOCKERFILE := deploy/docker/Dockerfile export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing ifeq ($(OS),Windows_NT) From bff41296aa140c7e4f2fcb34f35e11c571cba217 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 29 Dec 2021 16:56:24 +0800 Subject: [PATCH 47/99] fix(ecpool): update ecpool to 0.5.2 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 7b08d38a0..61a3f4510 100644 --- a/rebar.config +++ b/rebar.config @@ -56,7 +56,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.9"}}} - , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} + , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} , {replayq, "0.3.3"} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}} From ea1aaa9806a9f59aeef7641c8bf5209e61175c93 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 29 Dec 2021 16:58:02 +0800 Subject: [PATCH 48/99] fix(bridge): remove clientid config from MQTT bridges Don't allow the user provide the clientid for connecting the remote broker. We generate the clientid using the bridge id and node name. --- apps/emqx_bridge/src/emqx_bridge_api.erl | 4 ++-- apps/emqx_connector/src/emqx_connector_api.erl | 4 ++-- apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl | 6 +----- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 0f291ac1a..1358cccdc 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -431,8 +431,8 @@ rpc_multicall(Func, Args) -> end. filter_out_request_body(Conf) -> - ExtraConfs = [<<"id">>, <<"status">>, <<"node_status">>, <<"node_metrics">>, - <<"metrics">>, <<"node">>], + ExtraConfs = [<<"id">>, <<"status">>, <<"node_status">>, + <<"node_metrics">>, <<"metrics">>, <<"node">>], maps:without(ExtraConfs, Conf). rpc_call(Node, Fun, Args) -> diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index 5d6bddb6a..c20632658 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -218,7 +218,7 @@ schema("/connectors/:id") -> {400, error_msg('ALREADY_EXISTS', <<"connector already exists">>)}; {error, not_found} -> case emqx_connector:update(ConnType, ConnName, - maps:without([<<"type">>, <<"name">>], Params)) of + filter_out_request_body(Params)) of {ok, #{raw_config := RawConf}} -> Id = emqx_connector:connector_id(ConnType, ConnName), {201, format_resp(Id, RawConf)}; @@ -279,7 +279,7 @@ format_resp(ConnId, RawConf) -> }. filter_out_request_body(Conf) -> - ExtraConfs = [<<"num_of_bridges">>, <<"type">>, <<"name">>], + ExtraConfs = [<<"clientid">>, <<"num_of_bridges">>, <<"type">>, <<"name">>], maps:without(ExtraConfs, Conf). bin(S) when is_list(S) -> diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index 6fabb6b5d..bee1861bd 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -39,7 +39,7 @@ fields("config") -> fields("connector") -> [ {mode, - sc(hoconsc:enum([cluster_singleton, cluster_shareload]), + sc(hoconsc:enum([cluster_shareload]), #{ default => cluster_shareload , desc => """ The mode of the MQTT Bridge. Can be one of 'cluster_singleton' or 'cluster_shareload'
@@ -76,10 +76,6 @@ topic filters for 'remote_topic' of ingress connections. #{ default => "emqx" , desc => "The password of the MQTT protocol" })} - , {clientid, - sc(binary(), - #{ desc => "The clientid of the MQTT protocol" - })} , {clean_start, sc(boolean(), #{ default => true From c23436166b861e106d37b98d42d42916d0c5a086 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 29 Dec 2021 17:27:02 +0800 Subject: [PATCH 49/99] fix(bridge): HTTP connector should failed on non-200 status codes --- apps/emqx_connector/src/emqx_connector_http.erl | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index ac0847a91..2f72c869e 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -211,8 +211,20 @@ on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery, request => NRequest, reason => Reason, connector => InstId}), emqx_resource:query_failed(AfterQuery); - _ -> - emqx_resource:query_success(AfterQuery) + {ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 -> + emqx_resource:query_success(AfterQuery); + {ok, StatusCode, _, _} when StatusCode >= 200 andalso StatusCode < 300 -> + emqx_resource:query_success(AfterQuery); + {ok, StatusCode, _} -> + ?SLOG(error, #{msg => "http connector do reqeust, received error response", + request => NRequest, connector => InstId, + status_code => StatusCode}), + emqx_resource:query_failed(AfterQuery); + {ok, StatusCode, _, _} -> + ?SLOG(error, #{msg => "http connector do reqeust, received error response", + request => NRequest, connector => InstId, + status_code => StatusCode}), + emqx_resource:query_failed(AfterQuery) end, Result. From aefcd6275bae5e6744f129979bd76d7e4895a9e2 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 29 Dec 2021 17:40:56 +0800 Subject: [PATCH 50/99] fix(bridges): ingress MQTT bridges didn't increase counters on msg received --- .../src/emqx_connector_mqtt.erl | 24 ++++++++------ .../src/mqtt/emqx_connector_mqtt_mod.erl | 1 - .../src/mqtt/emqx_connector_mqtt_msg.erl | 32 +++++++++++++++++-- 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index a9647b2c1..c216e905c 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -29,7 +29,7 @@ , bridges/0 ]). --export([on_message_received/2]). +-export([on_message_received/3]). %% callbacks of behaviour emqx_resource -export([ on_start/2 @@ -105,14 +105,17 @@ drop_bridge(Name) -> case supervisor:terminate_child(?MODULE, Name) of ok -> supervisor:delete_child(?MODULE, Name); + {error, not_found} -> + ok; {error, Error} -> {error, Error} end. %% =================================================================== -%% When use this bridge as a data source, ?MODULE:on_message_received/2 will be called +%% When use this bridge as a data source, ?MODULE:on_message_received will be called %% if the bridge received msgs from the remote broker. -on_message_received(Msg, HookPoint) -> +on_message_received(Msg, HookPoint, InstId) -> + _ = emqx_resource:query(InstId, {message_received, Msg}), emqx:run_hook(HookPoint, [Msg]). %% =================================================================== @@ -123,8 +126,8 @@ on_start(InstId, Conf) -> BasicConf = basic_config(Conf), BridgeConf = BasicConf#{ name => InstanceId, - clientid => clientid(maps:get(clientid, Conf, InstId)), - subscriptions => make_sub_confs(maps:get(ingress, Conf, undefined)), + clientid => clientid(InstId), + subscriptions => make_sub_confs(maps:get(ingress, Conf, undefined), InstId), forwards => make_forward_confs(maps:get(egress, Conf, undefined)) }, case ?MODULE:create_bridge(BridgeConf) of @@ -149,6 +152,9 @@ on_stop(_InstId, #{name := InstanceId}) -> connector => InstanceId, reason => Reason}) end. +on_query(_InstId, {message_received, _Msg}, AfterQuery, _State) -> + emqx_resource:query_success(AfterQuery); + on_query(_InstId, {send_message, Msg}, AfterQuery, #{name := InstanceId}) -> ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}), emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg), @@ -166,15 +172,15 @@ ensure_mqtt_worker_started(InstanceId) -> {error, Reason} -> {error, Reason} end. -make_sub_confs(EmptyMap) when map_size(EmptyMap) == 0 -> +make_sub_confs(EmptyMap, _) when map_size(EmptyMap) == 0 -> undefined; -make_sub_confs(undefined) -> +make_sub_confs(undefined, _) -> undefined; -make_sub_confs(SubRemoteConf) -> +make_sub_confs(SubRemoteConf, InstId) -> case maps:take(hookpoint, SubRemoteConf) of error -> SubRemoteConf; {HookPoint, SubConf} -> - MFA = {?MODULE, on_message_received, [HookPoint]}, + MFA = {?MODULE, on_message_received, [HookPoint, InstId]}, SubConf#{on_message_received => MFA} end. diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl index 3ab410391..d7abcda84 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl @@ -168,7 +168,6 @@ handle_publish(Msg, undefined) -> handle_publish(Msg, Vars) -> ?SLOG(debug, #{msg => "publish_to_local_broker", message => Msg, vars => Vars}), - emqx_metrics:inc('bridge.mqtt.message_received_from_remote', 1), case Vars of #{on_message_received := {Mod, Func, Args}} -> _ = erlang:apply(Mod, Func, [Msg | Args]); diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl index 1357037ee..a0dd9eec1 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -61,7 +61,7 @@ make_pub_vars(Mountpoint, Conf) when is_map(Conf) -> -> exp_msg(). to_remote_msg(#message{flags = Flags0} = Msg, Vars) -> Retain0 = maps:get(retain, Flags0, false), - MapMsg = maps:put(retain, Retain0, emqx_message:to_map(Msg)), + MapMsg = maps:put(retain, Retain0, emqx_rule_events:eventmsg_publish(Msg)), to_remote_msg(MapMsg, Vars); to_remote_msg(MapMsg, #{remote_topic := TopicToken, payload := PayloadToken, remote_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) when is_map(MapMsg) -> @@ -78,9 +78,10 @@ to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) -> Msg#message{topic = topic(Mountpoint, Topic)}. %% published from remote node over a MQTT connection -to_broker_msg(#{dup := Dup, properties := Props} = MapMsg, +to_broker_msg(#{dup := Dup, properties := Props} = MapMsg0, #{local_topic := TopicToken, payload := PayloadToken, local_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) -> + MapMsg = format_msg_received(MapMsg0), Topic = replace_vars_in_str(TopicToken, MapMsg), Payload = process_payload(PayloadToken, MapMsg), QoS = replace_simple_var(QoSToken, MapMsg), @@ -89,6 +90,33 @@ to_broker_msg(#{dup := Dup, properties := Props} = MapMsg, emqx_message:set_flags(#{dup => Dup, retain => Retain}, emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload))). +format_msg_received(#{dup := Dup, payload := Payload, properties := Props, + qos := QoS, retain := Retain, topic := Topic}) -> + #{event => '$bridges/mqtt', + id => emqx_guid:to_hexstr(emqx_guid:gen()), + payload => Payload, + topic => Topic, + qos => QoS, + flags => #{dup => Dup, retain => Retain}, + pub_props => printable_maps(Props), + timestamp => erlang:system_time(millisecond), + node => node() + }. + +printable_maps(undefined) -> #{}; +printable_maps(Headers) -> + maps:fold( + fun ('User-Property', V0, AccIn) when is_list(V0) -> + AccIn#{ + 'User-Property' => maps:from_list(V0), + 'User-Property-Pairs' => [#{ + key => Key, + value => Value + } || {Key, Value} <- V0] + }; + (K, V0, AccIn) -> AccIn#{K => V0} + end, #{}, Headers). + process_payload([], Msg) -> emqx_json:encode(Msg); process_payload(Tks, Msg) -> From 14089a572e5f9d7929d7ee409b192853c24eb90d Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 29 Dec 2021 18:09:56 +0800 Subject: [PATCH 51/99] fix(bridge): changes timeouts from 30s to 15s --- apps/emqx_bridge/etc/emqx_bridge.conf | 4 ++-- apps/emqx_bridge/src/emqx_bridge_api.erl | 4 ++-- apps/emqx_bridge/src/emqx_bridge_http_schema.erl | 2 +- apps/emqx_connector/src/emqx_connector_api.erl | 4 ++-- apps/emqx_connector/src/emqx_connector_http.erl | 2 +- apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/emqx_bridge/etc/emqx_bridge.conf b/apps/emqx_bridge/etc/emqx_bridge.conf index 04f4709b8..de931ae12 100644 --- a/apps/emqx_bridge/etc/emqx_bridge.conf +++ b/apps/emqx_bridge/etc/emqx_bridge.conf @@ -34,8 +34,8 @@ # direction = egress # ## NOTE: we cannot use placehodler variables in the `scheme://host:port` part of the url # url = "http://localhost:9901/messages/${topic}" -# request_timeout = "30s" -# connect_timeout = "30s" +# request_timeout = "15s" +# connect_timeout = "15s" # max_retries = 3 # retry_interval = "10s" # pool_type = "random" diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 1358cccdc..4def97327 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -158,8 +158,8 @@ method_example(_Type, _Direction, put) -> info_example_basic(http, _) -> #{ url => <<"http://localhost:9901/messages/${topic}">>, - request_timeout => <<"30s">>, - connect_timeout => <<"30s">>, + request_timeout => <<"15s">>, + connect_timeout => <<"15s">>, max_retries => 3, retry_interval => <<"10s">>, pool_type => <<"random">>, diff --git a/apps/emqx_bridge/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge/src/emqx_bridge_http_schema.erl index 540a6a070..a5937509c 100644 --- a/apps/emqx_bridge/src/emqx_bridge_http_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_http_schema.erl @@ -59,7 +59,7 @@ Template with variables is allowed. """ })} , {request_timeout, mk(emqx_schema:duration_ms(), - #{ default => <<"30s">> + #{ default => <<"15s">> , desc =>""" How long will the HTTP request timeout. """ diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index c20632658..35e0c8ecb 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -107,14 +107,14 @@ info_example_basic(mqtt) -> #{ mode => cluster_shareload, server => <<"127.0.0.1:1883">>, - reconnect_interval => <<"30s">>, + reconnect_interval => <<"15s">>, proto_ver => <<"v4">>, username => <<"foo">>, password => <<"bar">>, clientid => <<"foo">>, clean_start => true, keepalive => <<"300s">>, - retry_interval => <<"30s">>, + retry_interval => <<"15s">>, max_inflight => 100, ssl => #{ enable => false diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 2f72c869e..509e293cf 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -75,7 +75,7 @@ For example: http://localhost:9901/ })} , {connect_timeout, sc(emqx_schema:duration_ms(), - #{ default => "30s" + #{ default => "15s" , desc => "The timeout when connecting to the HTTP server" })} , {max_retries, diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index bee1861bd..303617a29 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -60,7 +60,7 @@ topic filters for 'remote_topic' of ingress connections. #{ default => "127.0.0.1:1883" , desc => "The host and port of the remote MQTT broker" })} - , {reconnect_interval, mk_duration("reconnect interval", #{default => "30s"})} + , {reconnect_interval, mk_duration("reconnect interval", #{default => "15s"})} , {proto_ver, sc(hoconsc:enum([v3, v4, v5]), #{ default => v4 @@ -82,7 +82,7 @@ topic filters for 'remote_topic' of ingress connections. , desc => "The clean-start or the clean-session of the MQTT protocol" })} , {keepalive, mk_duration("keepalive", #{default => "300s"})} - , {retry_interval, mk_duration("retry interval", #{default => "30s"})} + , {retry_interval, mk_duration("retry interval", #{default => "15s"})} , {max_inflight, sc(integer(), #{ default => 32 From 110ae62b24407d7bb282c1aec0e884d5e0ccdc78 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 29 Dec 2021 18:41:50 +0800 Subject: [PATCH 52/99] fix(bridge): don't concat names into ids --- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- apps/emqx_connector/src/emqx_connector_api.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 4def97327..900ad99ee 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -276,7 +276,7 @@ schema("/bridges/:id/operation/:operation") -> '/bridges'(post, #{body := #{<<"type">> := BridgeType} = Conf0}) -> Conf = filter_out_request_body(Conf0), - BridgeName = maps:get(<<"name">>, Conf, emqx_misc:gen_id()), + BridgeName = emqx_misc:gen_id(), case emqx_bridge:lookup(BridgeType, BridgeName) of {ok, _} -> {400, error_msg('ALREADY_EXISTS', <<"bridge already exists">>)}; diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index 35e0c8ecb..82b63476d 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -212,7 +212,7 @@ schema("/connectors/:id") -> {200, [format_resp(Conn) || Conn <- emqx_connector:list()]}; '/connectors'(post, #{body := #{<<"type">> := ConnType} = Params}) -> - ConnName = maps:get(<<"name">>, Params, emqx_misc:gen_id()), + ConnName = emqx_misc:gen_id(), case emqx_connector:lookup(ConnType, ConnName) of {ok, _} -> {400, error_msg('ALREADY_EXISTS', <<"connector already exists">>)}; From d11cf6ad64531dd3620dcc1ba017d0433a0173d2 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 30 Dec 2021 12:31:13 +0800 Subject: [PATCH 53/99] fix(bridges): store connector name and bridge name to config files --- apps/emqx_bridge/src/emqx_bridge.erl | 5 +- apps/emqx_bridge/src/emqx_bridge_api.erl | 9 +- .../src/emqx_bridge_http_schema.erl | 4 + apps/emqx_bridge/src/emqx_bridge_schema.erl | 6 +- .../test/emqx_bridge_api_SUITE.erl | 116 +++++------- .../emqx_connector/src/emqx_connector_api.erl | 9 +- .../src/emqx_connector_http.erl | 2 +- .../src/emqx_connector_schema.erl | 9 + .../src/mqtt/emqx_connector_mqtt_schema.erl | 2 +- .../test/emqx_connector_api_SUITE.erl | 177 ++++++++---------- .../src/emqx_resource_instance.erl | 5 +- 11 files changed, 168 insertions(+), 176 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index a6681d3f1..d46ce217e 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -222,7 +222,10 @@ update(Type, Name, {OldConf, Conf}) -> true -> %% we don't need to recreate the bridge if this config change is only to %% toggole the config 'bridge.{type}.{name}.enable' - ok + case maps:get(enable, Conf, true) of + false -> stop(Type, Name); + true -> start(Type, Name) + end end. recreate(Type, Name) -> diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 900ad99ee..a5b9aa984 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -356,9 +356,8 @@ operation_to_conf_req(<<"restart">>) -> restart; operation_to_conf_req(_) -> invalid. ensure_bridge_created(BridgeType, BridgeName, Conf) -> - Conf1 = maps:without([<<"type">>, <<"name">>], Conf), case emqx_conf:update(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName], - Conf1, #{override_to => cluster}) of + Conf, #{override_to => cluster}) of {ok, _} -> ok; {error, Reason} -> {error, error_msg('BAD_ARG', Reason)} @@ -411,12 +410,12 @@ aggregate_metrics(AllMetrics) -> format_resp(#{id := Id, raw_config := RawConf, resource_data := #{status := Status, metrics := Metrics}}) -> - {Type, Name} = emqx_bridge:parse_bridge_id(Id), + {Type, BridgeName} = emqx_bridge:parse_bridge_id(Id), IsConnected = fun(started) -> connected; (_) -> disconnected end, RawConf#{ id => Id, type => Type, - name => Name, + name => maps:get(<<"name">>, RawConf, BridgeName), node => node(), status => IsConnected(Status), metrics => Metrics @@ -431,7 +430,7 @@ rpc_multicall(Func, Args) -> end. filter_out_request_body(Conf) -> - ExtraConfs = [<<"id">>, <<"status">>, <<"node_status">>, + ExtraConfs = [<<"id">>, <<"type">>, <<"status">>, <<"node_status">>, <<"node_metrics">>, <<"metrics">>, <<"node">>], maps:without(ExtraConfs, Conf). diff --git a/apps/emqx_bridge/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge/src/emqx_bridge_http_schema.erl index a5937509c..494911d21 100644 --- a/apps/emqx_bridge/src/emqx_bridge_http_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_http_schema.erl @@ -84,6 +84,10 @@ basic_config() -> #{ desc => "Enable or disable this bridge" , default => true })} + , {name, + mk(binary(), + #{ desc => "Bridge name, used as a human-readable description of the bridge." + })} , {direction, mk(egress, #{ desc => "The direction of this bridge, MUST be egress" diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl index 82fc79ebf..00d461098 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -43,9 +43,13 @@ http_schema(Method) -> common_bridge_fields() -> [ {enable, mk(boolean(), - #{ desc =>"Enable or disable this bridge" + #{ desc => "Enable or disable this bridge" , default => true })} + , {name, + mk(binary(), + #{ desc => "Bridge name, used as a human-readable description of the bridge." + })} , {connector, mk(binary(), #{ nullable => false diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 7724d467c..807ad32f6 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -23,12 +23,13 @@ -define(CONF_DEFAULT, <<"bridges: {}">>). -define(BRIDGE_TYPE, <<"http">>). -define(BRIDGE_NAME, <<"test_bridge">>). --define(BRIDGE_ID, <<"http:test_bridge">>). -define(URL(PORT, PATH), list_to_binary( io_lib:format("http://localhost:~s/~s", [integer_to_list(PORT), PATH]))). --define(HTTP_BRIDGE(URL), +-define(HTTP_BRIDGE(URL, TYPE, NAME), #{ + <<"type">> => TYPE, + <<"name">> => NAME, <<"url">> => URL, <<"local_topic">> => <<"emqx_http/#">>, <<"method">> => <<"post">>, @@ -145,32 +146,18 @@ t_http_crud_apis(_) -> %% POST /bridges/ will create a bridge URL1 = ?URL(Port, "path1"), {ok, 201, Bridge} = request(post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1)#{ - <<"type">> => ?BRIDGE_TYPE, - <<"name">> => ?BRIDGE_NAME - }), + ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)), %ct:pal("---bridge: ~p", [Bridge]), - ?assertMatch(#{ <<"id">> := ?BRIDGE_ID - , <<"type">> := ?BRIDGE_TYPE - , <<"name">> := ?BRIDGE_NAME - , <<"status">> := _ - , <<"node_status">> := [_|_] - , <<"metrics">> := _ - , <<"node_metrics">> := [_|_] - , <<"url">> := URL1 - }, jsx:decode(Bridge)), - - %% create a again returns an error - {ok, 400, RetMsg} = request(post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1)#{ - <<"type">> => ?BRIDGE_TYPE, - <<"name">> => ?BRIDGE_NAME - }), - ?assertMatch( - #{ <<"code">> := _ - , <<"message">> := <<"bridge already exists">> - }, jsx:decode(RetMsg)), + #{ <<"id">> := BridgeID + , <<"type">> := ?BRIDGE_TYPE + , <<"name">> := ?BRIDGE_NAME + , <<"status">> := _ + , <<"node_status">> := [_|_] + , <<"metrics">> := _ + , <<"node_metrics">> := [_|_] + , <<"url">> := URL1 + } = jsx:decode(Bridge), %% send an message to emqx and the message should be forwarded to the HTTP server Body = <<"my msg">>, @@ -188,9 +175,9 @@ t_http_crud_apis(_) -> end), %% update the request-path of the bridge URL2 = ?URL(Port, "path2"), - {ok, 200, Bridge2} = request(put, uri(["bridges", ?BRIDGE_ID]), - ?HTTP_BRIDGE(URL2)), - ?assertMatch(#{ <<"id">> := ?BRIDGE_ID + {ok, 200, Bridge2} = request(put, uri(["bridges", BridgeID]), + ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME)), + ?assertMatch(#{ <<"id">> := BridgeID , <<"type">> := ?BRIDGE_TYPE , <<"name">> := ?BRIDGE_NAME , <<"status">> := _ @@ -202,7 +189,7 @@ t_http_crud_apis(_) -> %% list all bridges again, assert Bridge2 is in it {ok, 200, Bridge2Str} = request(get, uri(["bridges"]), []), - ?assertMatch([#{ <<"id">> := ?BRIDGE_ID + ?assertMatch([#{ <<"id">> := BridgeID , <<"type">> := ?BRIDGE_TYPE , <<"name">> := ?BRIDGE_NAME , <<"status">> := _ @@ -213,8 +200,8 @@ t_http_crud_apis(_) -> }], jsx:decode(Bridge2Str)), %% get the bridge by id - {ok, 200, Bridge3Str} = request(get, uri(["bridges", ?BRIDGE_ID]), []), - ?assertMatch(#{ <<"id">> := ?BRIDGE_ID + {ok, 200, Bridge3Str} = request(get, uri(["bridges", BridgeID]), []), + ?assertMatch(#{ <<"id">> := BridgeID , <<"type">> := ?BRIDGE_TYPE , <<"name">> := ?BRIDGE_NAME , <<"status">> := _ @@ -238,12 +225,12 @@ t_http_crud_apis(_) -> end), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID]), []), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), %% update a deleted bridge returns an error - {ok, 404, ErrMsg2} = request(put, uri(["bridges", ?BRIDGE_ID]), - ?HTTP_BRIDGE(URL2)), + {ok, 404, ErrMsg2} = request(put, uri(["bridges", BridgeID]), + ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME)), ?assertMatch( #{ <<"code">> := _ , <<"message">> := <<"bridge not found">> @@ -251,52 +238,51 @@ t_http_crud_apis(_) -> ok. t_start_stop_bridges(_) -> + %% assert we there's no bridges at first + {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), + Port = start_http_server(fun handle_fun_200_ok/2), URL1 = ?URL(Port, "abc"), {ok, 201, Bridge} = request(post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1)#{ - <<"type">> => ?BRIDGE_TYPE, - <<"name">> => ?BRIDGE_NAME - }), + ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)), %ct:pal("the bridge ==== ~p", [Bridge]), - ?assertMatch( - #{ <<"id">> := ?BRIDGE_ID - , <<"type">> := ?BRIDGE_TYPE - , <<"name">> := ?BRIDGE_NAME - , <<"status">> := _ - , <<"node_status">> := [_|_] - , <<"metrics">> := _ - , <<"node_metrics">> := [_|_] - , <<"url">> := URL1 - }, jsx:decode(Bridge)), + #{ <<"id">> := BridgeID + , <<"type">> := ?BRIDGE_TYPE + , <<"name">> := ?BRIDGE_NAME + , <<"status">> := _ + , <<"node_status">> := [_|_] + , <<"metrics">> := _ + , <<"node_metrics">> := [_|_] + , <<"url">> := URL1 + } = jsx:decode(Bridge), %% stop it - {ok, 200, <<>>} = request(post, operation_path(stop), <<"">>), - {ok, 200, Bridge2} = request(get, uri(["bridges", ?BRIDGE_ID]), []), - ?assertMatch(#{ <<"id">> := ?BRIDGE_ID + {ok, 200, <<>>} = request(post, operation_path(stop, BridgeID), <<"">>), + {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []), + ?assertMatch(#{ <<"id">> := BridgeID , <<"status">> := <<"disconnected">> }, jsx:decode(Bridge2)), %% start again - {ok, 200, <<>>} = request(post, operation_path(start), <<"">>), - {ok, 200, Bridge3} = request(get, uri(["bridges", ?BRIDGE_ID]), []), - ?assertMatch(#{ <<"id">> := ?BRIDGE_ID + {ok, 200, <<>>} = request(post, operation_path(start, BridgeID), <<"">>), + {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []), + ?assertMatch(#{ <<"id">> := BridgeID , <<"status">> := <<"connected">> }, jsx:decode(Bridge3)), %% restart an already started bridge - {ok, 200, <<>>} = request(post, operation_path(restart), <<"">>), - {ok, 200, Bridge3} = request(get, uri(["bridges", ?BRIDGE_ID]), []), - ?assertMatch(#{ <<"id">> := ?BRIDGE_ID + {ok, 200, <<>>} = request(post, operation_path(restart, BridgeID), <<"">>), + {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []), + ?assertMatch(#{ <<"id">> := BridgeID , <<"status">> := <<"connected">> }, jsx:decode(Bridge3)), %% stop it again - {ok, 200, <<>>} = request(post, operation_path(stop), <<"">>), + {ok, 200, <<>>} = request(post, operation_path(stop, BridgeID), <<"">>), %% restart a stopped bridge - {ok, 200, <<>>} = request(post, operation_path(restart), <<"">>), - {ok, 200, Bridge4} = request(get, uri(["bridges", ?BRIDGE_ID]), []), - ?assertMatch(#{ <<"id">> := ?BRIDGE_ID + {ok, 200, <<>>} = request(post, operation_path(restart, BridgeID), <<"">>), + {ok, 200, Bridge4} = request(get, uri(["bridges", BridgeID]), []), + ?assertMatch(#{ <<"id">> := BridgeID , <<"status">> := <<"connected">> }, jsx:decode(Bridge4)), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID]), []), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []). %%-------------------------------------------------------------------- @@ -332,5 +318,5 @@ auth_header_() -> {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password), {"Authorization", "Bearer " ++ binary_to_list(Token)}. -operation_path(Oper) -> - uri(["bridges", ?BRIDGE_ID, "operation", Oper]). +operation_path(Oper, BridgeID) -> + uri(["bridges", BridgeID, "operation", Oper]). diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index 82b63476d..4989cf17e 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -155,8 +155,7 @@ schema("/connectors") -> }, post => #{ tags => [<<"connectors">>], - description => <<"Create a new connector by given Id
" - "The ID must be of format '{type}:{name}'">>, + description => <<"Create a new connector">>, summary => <<"Create connector">>, requestBody => post_request_body_schema(), responses => #{ @@ -270,16 +269,16 @@ format_resp(#{<<"id">> := Id} = RawConf) -> format_resp(ConnId, RawConf) -> NumOfBridges = length(emqx_bridge:list_bridges_by_connector(ConnId)), - {Type, Name} = emqx_connector:parse_connector_id(ConnId), + {Type, ConnName} = emqx_connector:parse_connector_id(ConnId), RawConf#{ <<"id">> => ConnId, <<"type">> => Type, - <<"name">> => Name, + <<"name">> => maps:get(<<"name">>, RawConf, ConnName), <<"num_of_bridges">> => NumOfBridges }. filter_out_request_body(Conf) -> - ExtraConfs = [<<"clientid">>, <<"num_of_bridges">>, <<"type">>, <<"name">>], + ExtraConfs = [<<"clientid">>, <<"num_of_bridges">>, <<"type">>], maps:without(ExtraConfs, Conf). bin(S) when is_list(S) -> diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 509e293cf..77d498c6b 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -112,7 +112,7 @@ If the request is provided, the caller can send HTTP requests via emqx_resource:query(ResourceId, {send_message, BridgeId, Message}) """ })} - ] ++ emqx_connector_schema_lib:ssl_fields(); + ] ++ emqx_connector_schema:common_fields() ++ emqx_connector_schema_lib:ssl_fields(); fields("request") -> [ {method, hoconsc:mk(hoconsc:enum([post, put, get, delete]), #{nullable => true})} diff --git a/apps/emqx_connector/src/emqx_connector_schema.erl b/apps/emqx_connector/src/emqx_connector_schema.erl index 33d10802b..ed663ee60 100644 --- a/apps/emqx_connector/src/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/emqx_connector_schema.erl @@ -8,6 +8,8 @@ -export([roots/0, fields/1]). +-export([common_fields/0]). + -export([ get_response/0 , put_request/0 , post_request/0 @@ -49,3 +51,10 @@ fields("connectors") -> schema_mod(Type) -> list_to_atom(lists:concat(["emqx_connector_", Type])). + +common_fields() -> + [ {name, + mk(binary(), + #{ desc => "Connector name, used as a human-readable description of the connector." + })} + ]. diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index 303617a29..44add053c 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -94,7 +94,7 @@ topic filters for 'remote_topic' of ingress connections. Queue messages in disk files. """ })} - ] ++ emqx_connector_schema_lib:ssl_fields(); + ] ++ emqx_connector_schema:common_fields() ++ emqx_connector_schema_lib:ssl_fields(); fields("ingress") -> %% the message maybe subscribed by rules, in this case 'local_topic' is not necessary diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index 307852546..1a96a3596 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -26,11 +26,8 @@ -define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>). -define(CONNECTR_TYPE, <<"mqtt">>). -define(CONNECTR_NAME, <<"test_connector">>). --define(CONNECTR_ID, <<"mqtt:test_connector">>). -define(BRIDGE_NAME_INGRESS, <<"ingress_test_bridge">>). -define(BRIDGE_NAME_EGRESS, <<"egress_test_bridge">>). --define(BRIDGE_ID_INGRESS, <<"mqtt:ingress_test_bridge">>). --define(BRIDGE_ID_EGRESS, <<"mqtt:egress_test_bridge">>). -define(MQTT_CONNECOTR(Username), #{ <<"server">> => <<"127.0.0.1:1883">>, @@ -123,32 +120,21 @@ t_mqtt_crud_apis(_) -> , <<"name">> => ?CONNECTR_NAME }), - %ct:pal("---connector: ~p", [Connector]), - ?assertMatch(#{ <<"id">> := ?CONNECTR_ID - , <<"type">> := ?CONNECTR_TYPE - , <<"name">> := ?CONNECTR_NAME - , <<"server">> := <<"127.0.0.1:1883">> - , <<"username">> := User1 - , <<"password">> := <<"">> - , <<"proto_ver">> := <<"v4">> - , <<"ssl">> := #{<<"enable">> := false} - }, jsx:decode(Connector)), - - %% create a again returns an error - {ok, 400, RetMsg} = request(post, uri(["connectors"]), - ?MQTT_CONNECOTR(User1)#{ <<"type">> => ?CONNECTR_TYPE - , <<"name">> => ?CONNECTR_NAME - }), - ?assertMatch( - #{ <<"code">> := _ - , <<"message">> := <<"connector already exists">> - }, jsx:decode(RetMsg)), + #{ <<"id">> := ConnctorID + , <<"type">> := ?CONNECTR_TYPE + , <<"name">> := ?CONNECTR_NAME + , <<"server">> := <<"127.0.0.1:1883">> + , <<"username">> := User1 + , <<"password">> := <<"">> + , <<"proto_ver">> := <<"v4">> + , <<"ssl">> := #{<<"enable">> := false} + } = jsx:decode(Connector), %% update the request-path of the connector User2 = <<"user2">>, - {ok, 200, Connector2} = request(put, uri(["connectors", ?CONNECTR_ID]), + {ok, 200, Connector2} = request(put, uri(["connectors", ConnctorID]), ?MQTT_CONNECOTR(User2)), - ?assertMatch(#{ <<"id">> := ?CONNECTR_ID + ?assertMatch(#{ <<"id">> := ConnctorID , <<"server">> := <<"127.0.0.1:1883">> , <<"username">> := User2 , <<"password">> := <<"">> @@ -158,7 +144,7 @@ t_mqtt_crud_apis(_) -> %% list all connectors again, assert Connector2 is in it {ok, 200, Connector2Str} = request(get, uri(["connectors"]), []), - ?assertMatch([#{ <<"id">> := ?CONNECTR_ID + ?assertMatch([#{ <<"id">> := ConnctorID , <<"type">> := ?CONNECTR_TYPE , <<"name">> := ?CONNECTR_NAME , <<"server">> := <<"127.0.0.1:1883">> @@ -169,8 +155,8 @@ t_mqtt_crud_apis(_) -> }], jsx:decode(Connector2Str)), %% get the connector by id - {ok, 200, Connector3Str} = request(get, uri(["connectors", ?CONNECTR_ID]), []), - ?assertMatch(#{ <<"id">> := ?CONNECTR_ID + {ok, 200, Connector3Str} = request(get, uri(["connectors", ConnctorID]), []), + ?assertMatch(#{ <<"id">> := ConnctorID , <<"type">> := ?CONNECTR_TYPE , <<"name">> := ?CONNECTR_NAME , <<"server">> := <<"127.0.0.1:1883">> @@ -181,11 +167,11 @@ t_mqtt_crud_apis(_) -> }, jsx:decode(Connector3Str)), %% delete the connector - {ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []), + {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []), {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []), %% update a deleted connector returns an error - {ok, 404, ErrMsg2} = request(put, uri(["connectors", ?CONNECTR_ID]), + {ok, 404, ErrMsg2} = request(put, uri(["connectors", ConnctorID]), ?MQTT_CONNECOTR(User2)), ?assertMatch( #{ <<"code">> := _ @@ -205,28 +191,28 @@ t_mqtt_conn_bridge_ingress(_) -> , <<"name">> => ?CONNECTR_NAME }), - ?assertMatch(#{ <<"id">> := ?CONNECTR_ID - , <<"server">> := <<"127.0.0.1:1883">> - , <<"num_of_bridges">> := 0 - , <<"username">> := User1 - , <<"password">> := <<"">> - , <<"proto_ver">> := <<"v4">> - , <<"ssl">> := #{<<"enable">> := false} - }, jsx:decode(Connector)), + #{ <<"id">> := ConnctorID + , <<"server">> := <<"127.0.0.1:1883">> + , <<"num_of_bridges">> := 0 + , <<"username">> := User1 + , <<"password">> := <<"">> + , <<"proto_ver">> := <<"v4">> + , <<"ssl">> := #{<<"enable">> := false} + } = jsx:decode(Connector), %% ... and a MQTT bridge, using POST %% we bind this bridge to the connector created just now {ok, 201, Bridge} = request(post, uri(["bridges"]), - ?MQTT_BRIDGE_INGRESS(?CONNECTR_ID)#{ + ?MQTT_BRIDGE_INGRESS(ConnctorID)#{ <<"type">> => ?CONNECTR_TYPE, <<"name">> => ?BRIDGE_NAME_INGRESS }), - ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_INGRESS - , <<"type">> := <<"mqtt">> - , <<"status">> := <<"connected">> - , <<"connector">> := ?CONNECTR_ID - }, jsx:decode(Bridge)), + #{ <<"id">> := BridgeIDIngress + , <<"type">> := <<"mqtt">> + , <<"status">> := <<"connected">> + , <<"connector">> := ConnctorID + } = jsx:decode(Bridge), %% we now test if the bridge works as expected @@ -252,17 +238,17 @@ t_mqtt_conn_bridge_ingress(_) -> end), %% get the connector by id, verify the num_of_bridges now is 1 - {ok, 200, Connector1Str} = request(get, uri(["connectors", ?CONNECTR_ID]), []), - ?assertMatch(#{ <<"id">> := ?CONNECTR_ID + {ok, 200, Connector1Str} = request(get, uri(["connectors", ConnctorID]), []), + ?assertMatch(#{ <<"id">> := ConnctorID , <<"num_of_bridges">> := 1 }, jsx:decode(Connector1Str)), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_INGRESS]), []), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), %% delete the connector - {ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []), + {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []), {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []), ok. @@ -279,29 +265,28 @@ t_mqtt_conn_bridge_egress(_) -> }), %ct:pal("---connector: ~p", [Connector]), - ?assertMatch(#{ <<"id">> := ?CONNECTR_ID - , <<"server">> := <<"127.0.0.1:1883">> - , <<"username">> := User1 - , <<"password">> := <<"">> - , <<"proto_ver">> := <<"v4">> - , <<"ssl">> := #{<<"enable">> := false} - }, jsx:decode(Connector)), + #{ <<"id">> := ConnctorID + , <<"server">> := <<"127.0.0.1:1883">> + , <<"username">> := User1 + , <<"password">> := <<"">> + , <<"proto_ver">> := <<"v4">> + , <<"ssl">> := #{<<"enable">> := false} + } = jsx:decode(Connector), %% ... and a MQTT bridge, using POST %% we bind this bridge to the connector created just now {ok, 201, Bridge} = request(post, uri(["bridges"]), - ?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{ + ?MQTT_BRIDGE_EGRESS(ConnctorID)#{ <<"type">> => ?CONNECTR_TYPE, <<"name">> => ?BRIDGE_NAME_EGRESS }), - %ct:pal("---bridge: ~p", [Bridge]), - ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS - , <<"type">> := ?CONNECTR_TYPE - , <<"name">> := ?BRIDGE_NAME_EGRESS - , <<"status">> := <<"connected">> - , <<"connector">> := ?CONNECTR_ID - }, jsx:decode(Bridge)), + #{ <<"id">> := BridgeIDEgress + , <<"type">> := ?CONNECTR_TYPE + , <<"name">> := ?BRIDGE_NAME_EGRESS + , <<"status">> := <<"connected">> + , <<"connector">> := ConnctorID + } = jsx:decode(Bridge), %% we now test if the bridge works as expected LocalTopic = <<"local_topic/1">>, @@ -326,19 +311,19 @@ t_mqtt_conn_bridge_egress(_) -> end), %% verify the metrics of the bridge - {ok, 200, BridgeStr} = request(get, uri(["bridges", ?BRIDGE_ID_EGRESS]), []), - ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS + {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []), + ?assertMatch(#{ <<"id">> := BridgeIDEgress , <<"metrics">> := ?metrics(1, 1, 0, _, _, _) , <<"node_metrics">> := [#{<<"node">> := _, <<"metrics">> := ?metrics(1, 1, 0, _, _, _)}] }, jsx:decode(BridgeStr)), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_EGRESS]), []), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), %% delete the connector - {ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []), + {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []), {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []), ok. @@ -358,37 +343,37 @@ t_mqtt_conn_update(_) -> }), %ct:pal("---connector: ~p", [Connector]), - ?assertMatch(#{ <<"id">> := ?CONNECTR_ID - , <<"server">> := <<"127.0.0.1:1883">> - }, jsx:decode(Connector)), + #{ <<"id">> := ConnctorID + , <<"server">> := <<"127.0.0.1:1883">> + } = jsx:decode(Connector), %% ... and a MQTT bridge, using POST %% we bind this bridge to the connector created just now {ok, 201, Bridge} = request(post, uri(["bridges"]), - ?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{ + ?MQTT_BRIDGE_EGRESS(ConnctorID)#{ <<"type">> => ?CONNECTR_TYPE, <<"name">> => ?BRIDGE_NAME_EGRESS }), - ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS - , <<"type">> := <<"mqtt">> - , <<"name">> := ?BRIDGE_NAME_EGRESS - , <<"status">> := <<"connected">> - , <<"connector">> := ?CONNECTR_ID - }, jsx:decode(Bridge)), + #{ <<"id">> := BridgeIDEgress + , <<"type">> := <<"mqtt">> + , <<"name">> := ?BRIDGE_NAME_EGRESS + , <<"status">> := <<"connected">> + , <<"connector">> := ConnctorID + } = jsx:decode(Bridge), %% then we try to update 'server' of the connector, to an unavailable IP address %% the update should fail because of 'unreachable' or 'connrefused' - {ok, 400, _ErrorMsg} = request(put, uri(["connectors", ?CONNECTR_ID]), + {ok, 400, _ErrorMsg} = request(put, uri(["connectors", ConnctorID]), ?MQTT_CONNECOTR2(<<"127.0.0.1:2603">>)), %% we fix the 'server' parameter to a normal one, it should work - {ok, 200, _} = request(put, uri(["connectors", ?CONNECTR_ID]), + {ok, 200, _} = request(put, uri(["connectors", ConnctorID]), ?MQTT_CONNECOTR2(<<"127.0.0.1 : 1883">>)), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_EGRESS]), []), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), %% delete the connector - {ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []), + {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []), {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []). t_mqtt_conn_update2(_) -> @@ -404,36 +389,36 @@ t_mqtt_conn_update2(_) -> , <<"name">> => ?CONNECTR_NAME }), - ?assertMatch(#{ <<"id">> := ?CONNECTR_ID - , <<"server">> := <<"127.0.0.1:2603">> - }, jsx:decode(Connector)), + #{ <<"id">> := ConnctorID + , <<"server">> := <<"127.0.0.1:2603">> + } = jsx:decode(Connector), %% ... and a MQTT bridge, using POST %% we bind this bridge to the connector created just now {ok, 201, Bridge} = request(post, uri(["bridges"]), - ?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{ + ?MQTT_BRIDGE_EGRESS(ConnctorID)#{ <<"type">> => ?CONNECTR_TYPE, <<"name">> => ?BRIDGE_NAME_EGRESS }), - ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS - , <<"type">> := <<"mqtt">> - , <<"name">> := ?BRIDGE_NAME_EGRESS - , <<"status">> := <<"disconnected">> - , <<"connector">> := ?CONNECTR_ID - }, jsx:decode(Bridge)), + #{ <<"id">> := BridgeIDEgress + , <<"type">> := <<"mqtt">> + , <<"name">> := ?BRIDGE_NAME_EGRESS + , <<"status">> := <<"disconnected">> + , <<"connector">> := ConnctorID + } = jsx:decode(Bridge), %% we fix the 'server' parameter to a normal one, it should work - {ok, 200, _} = request(put, uri(["connectors", ?CONNECTR_ID]), + {ok, 200, _} = request(put, uri(["connectors", ConnctorID]), ?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)), - {ok, 200, BridgeStr} = request(get, uri(["bridges", ?BRIDGE_ID_EGRESS]), []), - ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS + {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []), + ?assertMatch(#{ <<"id">> := BridgeIDEgress , <<"status">> := <<"connected">> }, jsx:decode(BridgeStr)), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_EGRESS]), []), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), %% delete the connector - {ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []), + {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []), {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []). t_mqtt_conn_testing(_) -> diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index 497affa5e..ebc812805 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -216,7 +216,10 @@ do_remove(Mod, InstId, ResourceState) -> do_restart(InstId) -> case lookup(InstId) of {ok, #{mod := Mod, state := ResourceState, config := Config} = Data} -> - _ = emqx_resource:call_stop(InstId, Mod, ResourceState), + _ = case ResourceState of + undefine -> ok; + _ -> emqx_resource:call_stop(InstId, Mod, ResourceState) + end, case emqx_resource:call_start(InstId, Mod, Config) of {ok, NewResourceState} -> ets:insert(emqx_resource_instance, From 9d733c2ec51bcca1bdfa47c2c911ee009fe27dd3 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 30 Dec 2021 14:40:28 +0800 Subject: [PATCH 54/99] fix(resource): typos on restart a resource --- apps/emqx_connector/src/emqx_connector_http.erl | 2 +- apps/emqx_connector/src/emqx_connector_schema.erl | 9 --------- .../src/mqtt/emqx_connector_mqtt_schema.erl | 7 ++++++- apps/emqx_resource/src/emqx_resource_instance.erl | 2 +- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 77d498c6b..509e293cf 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -112,7 +112,7 @@ If the request is provided, the caller can send HTTP requests via emqx_resource:query(ResourceId, {send_message, BridgeId, Message}) """ })} - ] ++ emqx_connector_schema:common_fields() ++ emqx_connector_schema_lib:ssl_fields(); + ] ++ emqx_connector_schema_lib:ssl_fields(); fields("request") -> [ {method, hoconsc:mk(hoconsc:enum([post, put, get, delete]), #{nullable => true})} diff --git a/apps/emqx_connector/src/emqx_connector_schema.erl b/apps/emqx_connector/src/emqx_connector_schema.erl index ed663ee60..33d10802b 100644 --- a/apps/emqx_connector/src/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/emqx_connector_schema.erl @@ -8,8 +8,6 @@ -export([roots/0, fields/1]). --export([common_fields/0]). - -export([ get_response/0 , put_request/0 , post_request/0 @@ -51,10 +49,3 @@ fields("connectors") -> schema_mod(Type) -> list_to_atom(lists:concat(["emqx_connector_", Type])). - -common_fields() -> - [ {name, - mk(binary(), - #{ desc => "Connector name, used as a human-readable description of the connector." - })} - ]. diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index 44add053c..b3484f5d9 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -55,6 +55,11 @@ clientid conflicts between different nodes. And we can only use shared subscript topic filters for 'remote_topic' of ingress connections. """ })} + , {name, + sc(binary(), + #{ nullable => true + , desc => "Connector name, used as a human-readable description of the connector." + })} , {server, sc(emqx_schema:ip_port(), #{ default => "127.0.0.1:1883" @@ -94,7 +99,7 @@ topic filters for 'remote_topic' of ingress connections. Queue messages in disk files. """ })} - ] ++ emqx_connector_schema:common_fields() ++ emqx_connector_schema_lib:ssl_fields(); + ] ++ emqx_connector_schema_lib:ssl_fields(); fields("ingress") -> %% the message maybe subscribed by rules, in this case 'local_topic' is not necessary diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index ebc812805..c23d55511 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -217,7 +217,7 @@ do_restart(InstId) -> case lookup(InstId) of {ok, #{mod := Mod, state := ResourceState, config := Config} = Data} -> _ = case ResourceState of - undefine -> ok; + undefined -> ok; _ -> emqx_resource:call_stop(InstId, Mod, ResourceState) end, case emqx_resource:call_start(InstId, Mod, Config) of From e2d899ad6e0f0ad6117c31c89151ab8e6adacf83 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 30 Dec 2021 15:30:41 +0800 Subject: [PATCH 55/99] fix(bridge): HTTP reqeust crash if using GET an DELETE method --- apps/emqx_connector/src/emqx_connector_http.erl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 509e293cf..8b366070d 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -201,7 +201,7 @@ on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery, #{pool_name := PoolName, base_path := BasePath} = State) -> ?TRACE("QUERY", "http_connector_received", #{request => Request, connector => InstId, state => State}), - NRequest = update_path(BasePath, Request), + NRequest = formalize_request(Method, BasePath, Request), case Result = ehttpc:request(case KeyOrNum of undefined -> PoolName; _ -> {PoolName, KeyOrNum} @@ -310,10 +310,14 @@ check_ssl_opts(URLFrom, Conf) -> {_, _} -> false end. -update_path(BasePath, {Path, Headers}) -> - {filename:join(BasePath, Path), Headers}; -update_path(BasePath, {Path, Headers, Body}) -> - {filename:join(BasePath, Path), Headers, Body}. +formalize_request(Method, BasePath, {Path, Headers, _Body}) + when Method =:= get; Method =:= delete -> + formalize_request(Method, BasePath, {Path, Headers}); +formalize_request(_Method, BasePath, {Path, Headers, Body}) -> + {filename:join(BasePath, Path), Headers, Body}; + +formalize_request(_Method, BasePath, {Path, Headers}) -> + {filename:join(BasePath, Path), Headers}. bin(Bin) when is_binary(Bin) -> Bin; From ea2d4674dfbb445495827fc372659ed92b590c06 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 30 Dec 2021 15:45:13 +0800 Subject: [PATCH 56/99] fix(resource): metrics were cleared after updating the resource --- apps/emqx_resource/src/emqx_resource_instance.erl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index c23d55511..745b8b684 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -149,7 +149,7 @@ do_recreate(InstId, ResourceType, NewConfig, Params) -> TestInstId = iolist_to_binary(emqx_misc:gen_id(16)), case do_create_dry_run(TestInstId, ResourceType, Config) of ok -> - do_remove(ResourceType, InstId, ResourceState), + do_remove(ResourceType, InstId, ResourceState, false), do_create(InstId, ResourceType, Config, #{force_create => true}); Error -> Error @@ -208,10 +208,15 @@ do_remove(InstId) -> end. do_remove(Mod, InstId, ResourceState) -> + do_remove(Mod, InstId, ResourceState, true). + +do_remove(Mod, InstId, ResourceState, ClearMetrics) -> _ = emqx_resource:call_stop(InstId, Mod, ResourceState), ets:delete(emqx_resource_instance, InstId), - ok = emqx_plugin_libs_metrics:clear_metrics(resource_metrics, InstId), - ok. + case ClearMetrics of + true -> ok = emqx_plugin_libs_metrics:clear_metrics(resource_metrics, InstId); + false -> ok + end. do_restart(InstId) -> case lookup(InstId) of From a42ab3d9dabf596fc6963947d34ef31e25297caf Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 30 Dec 2021 19:14:09 +0800 Subject: [PATCH 57/99] fix(rule): use emqx_conf:update/3 to make changes to all nodes --- apps/emqx_rule_engine/src/emqx_rule_engine_api.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index cbfda16db..d9138ffd1 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -172,7 +172,7 @@ param_path_id() -> {ok, _Rule} -> {400, #{code => 'BAD_ARGS', message => <<"rule id already exists">>}}; not_found -> - case emqx:update_config(ConfPath, Params, #{}) of + case emqx_conf:update(ConfPath, Params, #{}) of {ok, #{post_config_update := #{emqx_rule_engine := AllRules}}} -> [Rule] = get_one_rule(AllRules, Id), {201, format_rule_resp(Rule)}; @@ -200,7 +200,7 @@ param_path_id() -> '/rules/:id'(put, #{bindings := #{id := Id}, body := Params0}) -> Params = filter_out_request_body(Params0), ConfPath = emqx_rule_engine:config_key_path() ++ [Id], - case emqx:update_config(ConfPath, Params, #{}) of + case emqx_conf:update(ConfPath, Params, #{}) of {ok, #{post_config_update := #{emqx_rule_engine := AllRules}}} -> [Rule] = get_one_rule(AllRules, Id), {200, format_rule_resp(Rule)}; From 626a4c471391af4a766ef41016ce719b60cd8cd1 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 30 Dec 2021 21:53:32 +0800 Subject: [PATCH 58/99] fix(machine): some apps not restarted after joining into the cluster --- apps/emqx_machine/src/emqx_machine_boot.erl | 29 ++++++++++++++------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 24541990b..cffb914d8 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -96,7 +96,6 @@ reboot_apps() -> , emqx_resource , emqx_rule_engine , emqx_bridge - , emqx_bridge_mqtt , emqx_plugin_libs , emqx_management , emqx_retainer @@ -112,17 +111,17 @@ sorted_reboot_apps() -> app_deps(App) -> case application:get_key(App, applications) of - undefined -> []; + undefined -> undefined; {ok, List} -> lists:filter(fun(A) -> lists:member(A, reboot_apps()) end, List) end. sorted_reboot_apps(Apps) -> G = digraph:new(), try - lists:foreach(fun({App, Deps}) -> add_app(G, App, Deps) end, Apps), + NoDepApps = add_apps_to_digraph(G, Apps), case digraph_utils:topsort(G) of Sorted when is_list(Sorted) -> - Sorted; + Sorted ++ (NoDepApps -- Sorted); false -> Loops = find_loops(G), error({circular_application_dependency, Loops}) @@ -131,17 +130,29 @@ sorted_reboot_apps(Apps) -> digraph:delete(G) end. -add_app(G, App, undefined) -> +add_apps_to_digraph(G, Apps) -> + lists:foldl(fun + ({App, undefined}, Acc) -> + ?SLOG(debug, #{msg => "app_is_not_loaded", app => App}), + Acc; + ({App, []}, Acc) -> + Acc ++ [App]; %% use '++' to keep the original order + ({App, Deps}, Acc) -> + add_app_deps_to_digraph(G, App, Deps), + Acc + end, [], Apps). + +add_app_deps_to_digraph(G, App, undefined) -> ?SLOG(debug, #{msg => "app_is_not_loaded", app => App}), %% not loaded - add_app(G, App, []); -add_app(_G, _App, []) -> + add_app_deps_to_digraph(G, App, []); +add_app_deps_to_digraph(_G, _App, []) -> ok; -add_app(G, App, [Dep | Deps]) -> +add_app_deps_to_digraph(G, App, [Dep | Deps]) -> digraph:add_vertex(G, App), digraph:add_vertex(G, Dep), digraph:add_edge(G, Dep, App), %% dep -> app as dependency - add_app(G, App, Deps). + add_app_deps_to_digraph(G, App, Deps). find_loops(G) -> lists:filtermap( From 94a596556095be2dd588035a3f81ef005dd97126 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 30 Dec 2021 23:46:50 +0800 Subject: [PATCH 59/99] fix(rule): dead lock when update configs for rules --- apps/emqx_machine/test/emqx_machine_SUITE.erl | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.erl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_machine/test/emqx_machine_SUITE.erl b/apps/emqx_machine/test/emqx_machine_SUITE.erl index 03d9e6ba9..a760d2f5f 100644 --- a/apps/emqx_machine/test/emqx_machine_SUITE.erl +++ b/apps/emqx_machine/test/emqx_machine_SUITE.erl @@ -43,7 +43,7 @@ init_per_suite(Config) -> %% application:unload(emqx_authz), - emqx_common_test_helpers:start_apps([]), + emqx_common_test_helpers:start_apps([emqx_conf]), Config. end_per_suite(_Config) -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 5316ca5ef..6a579cbb0 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -187,11 +187,11 @@ init([]) -> {ok, #{}}. handle_call({insert_rule, Rule}, _From, State) -> - _ = emqx_plugin_libs_rule:cluster_call(?MODULE, do_insert_rule, [Rule]), + do_insert_rule(Rule), {reply, ok, State}; handle_call({delete_rule, Rule}, _From, State) -> - _ = emqx_plugin_libs_rule:cluster_call(?MODULE, do_delete_rule, [Rule]), + do_delete_rule(Rule), {reply, ok, State}; handle_call(Req, _From, State) -> From 86001765cb5b7acbb0025d2690753955e216890b Mon Sep 17 00:00:00 2001 From: lafirest Date: Thu, 30 Dec 2021 14:40:25 +0800 Subject: [PATCH 60/99] fix(emqx_retainer): fix config update error --- apps/emqx_retainer/src/emqx_retainer.erl | 26 +++++++++++------- apps/emqx_retainer/src/emqx_retainer_api.erl | 2 +- .../test/emqx_retainer_SUITE.erl | 27 +++++++++++++++++++ 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 9be449b60..e1780cc08 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -36,7 +36,8 @@ , update_config/1 , clean/0 , delete/1 - , page_read/3]). + , page_read/3 + , post_config_update/5]). %% gen_server callbacks -export([ init/1 @@ -165,24 +166,31 @@ get_expiry_time(#message{timestamp = Ts}) -> get_stop_publish_clear_msg() -> emqx_conf:get([?APP, stop_publish_clear_msg], false). --spec update_config(hocon:config()) -> ok. +-spec update_config(hocon:config()) -> {ok, _} | {error, _}. update_config(Conf) -> - gen_server:call(?MODULE, {?FUNCTION_NAME, Conf}). + emqx_conf:update([emqx_retainer], Conf, #{override_to => cluster}). clean() -> - gen_server:call(?MODULE, ?FUNCTION_NAME). + call(?FUNCTION_NAME). delete(Topic) -> - gen_server:call(?MODULE, {?FUNCTION_NAME, Topic}). + call({?FUNCTION_NAME, Topic}). page_read(Topic, Page, Limit) -> - gen_server:call(?MODULE, {?FUNCTION_NAME, Topic, Page, Limit}). + call({?FUNCTION_NAME, Topic, Page, Limit}). + +post_config_update(_, _UpdateReq, NewConf, OldConf, _AppEnvs) -> + call({update_config, NewConf, OldConf}). + +call(Req) -> + gen_server:call(?MODULE, Req, infinity). %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- init([]) -> + emqx_conf:add_handler([emqx_retainer], ?MODULE), init_shared_context(), State = new_state(), #{enable := Enable} = Cfg = emqx:get_config([?APP]), @@ -194,9 +202,7 @@ init([]) -> State end}. -handle_call({update_config, Conf}, _, State) -> - OldConf = emqx:get_config([?APP]), - {ok, #{config := NewConf}} = emqx:update_config([?APP], Conf), +handle_call({update_config, NewConf, OldConf}, _, State) -> State2 = update_config(State, NewConf, OldConf), {reply, ok, State2}; @@ -326,7 +332,7 @@ require_semaphore(Semaphore, Id) -> -spec wait_semaphore(non_neg_integer(), pos_integer()) -> boolean(). wait_semaphore(X, Id) when X < 0 -> - gen_server:call(?MODULE, {?FUNCTION_NAME, Id}, infinity); + call({?FUNCTION_NAME, Id}); wait_semaphore(_, _) -> true. diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index 5424629d9..b60ac5627 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -130,7 +130,7 @@ config(get, _) -> config(put, #{body := Body}) -> try - ok = emqx_retainer:update_config(Body), + {ok, _} = emqx_retainer:update_config(Body), {200, emqx:get_raw_config([emqx_retainer])} catch _:Reason:_ -> {400, diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index 7191bacc0..51db25ab2 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -20,6 +20,7 @@ -compile(nowarn_export_all). -define(APP, emqx_retainer). +-define(CLUSTER_RPC_SHARD, emqx_cluster_rpc_shard). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -49,13 +50,39 @@ emqx_retainer { %%-------------------------------------------------------------------- init_per_suite(Config) -> + application:load(emqx_conf), + ok = ekka:start(), + ok = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], infinity), + meck:new(emqx_alarm, [non_strict, passthrough, no_link]), + meck:expect(emqx_alarm, activate, 3, ok), + meck:expect(emqx_alarm, deactivate, 3, ok), + ok = emqx_config:init_load(emqx_retainer_schema, ?BASE_CONF), emqx_common_test_helpers:start_apps([emqx_retainer]), Config. end_per_suite(_Config) -> + ekka:stop(), + mria:stop(), + mria_mnesia:delete_schema(), + meck:unload(emqx_alarm), + emqx_common_test_helpers:stop_apps([emqx_retainer]). +init_per_testcase(_, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(), + timer:sleep(200), + Config. + +end_per_testcase(_, Config) -> + case erlang:whereis(node()) of + undefined -> ok; + P -> + erlang:unlink(P), + erlang:exit(P, kill) + end, + Config. + %%-------------------------------------------------------------------- %% Test Cases %%-------------------------------------------------------------------- From 4ebe47974d41178521ad47641d09c5fe6d00fe42 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 31 Dec 2021 10:51:29 +0800 Subject: [PATCH 61/99] chore(test): update api test script version --- .github/workflows/run_api_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml index 45b387c0d..796bcf451 100644 --- a/.github/workflows/run_api_tests.yaml +++ b/.github/workflows/run_api_tests.yaml @@ -61,7 +61,7 @@ jobs: - uses: actions/checkout@v2 with: repository: emqx/emqx-fvt - ref: 1.0.2-dev1 + ref: 1.0.3-dev1 path: . - uses: actions/setup-java@v1 with: @@ -93,7 +93,7 @@ jobs: run: | /opt/jmeter/bin/jmeter.sh \ -Jjmeter.save.saveservice.output_format=xml -n \ - -t .ci/api-test-suite/${{ matrix.script_name }}.jmx \ + -t api-test-suite/${{ matrix.script_name }}.jmx \ -Demqx_ip="127.0.0.1" \ -l jmeter_logs/${{ matrix.script_name }}.jtl \ -j jmeter_logs/logs/${{ matrix.script_name }}.log From 9ba454a63d3cc75686112251f6f553d2d323f67c Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 31 Dec 2021 12:00:43 +0800 Subject: [PATCH 62/99] fix(bridge): filter the topic of received msgs got from remote MQTT broker --- .../src/mqtt/emqx_connector_mqtt_mod.erl | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl index d7abcda84..30a1ccb30 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl @@ -173,15 +173,27 @@ handle_publish(Msg, Vars) -> _ = erlang:apply(Mod, Func, [Msg | Args]); _ -> ok end, - case maps:get(local_topic, Vars, undefined) of - undefined -> ok; - _Topic -> - emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars)) - end. + maybe_publish_to_local_broker(Msg, Vars). handle_disconnected(Reason, Parent) -> Parent ! {disconnected, self(), Reason}. +maybe_publish_to_local_broker(#{topic := Topic} = Msg, #{remote_topic := SubTopic} = Vars) -> + case maps:get(local_topic, Vars, undefined) of + undefined -> + %% local topic is not set, discard it + ok; + _ -> + case emqx_topic:match(Topic, SubTopic) of + true -> + _ = emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars)), + ok; + false -> + ?SLOG(warning, #{msg => "discard_message_as_topic_not_matched", + message => Msg, subscribed => SubTopic, got_topic => Topic}) + end + end. + make_hdlr(Parent, Vars) -> #{puback => {fun ?MODULE:handle_puback/2, [Parent]}, publish => {fun ?MODULE:handle_publish/2, [Vars]}, From f19ccdfcde868eda1b17375e13cd93a463c159ca Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 30 Dec 2021 16:42:50 +0800 Subject: [PATCH 63/99] fix(auto_subscribe): update config in cluster --- .../src/emqx_auto_subscribe.erl | 16 ++++++++++++++-- .../src/emqx_auto_subscribe_api.erl | 8 ++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl index 558e5005c..564772fe4 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl @@ -16,6 +16,8 @@ -module(emqx_auto_subscribe). +-include_lib("emqx/include/logger.hrl"). + -define(HOOK_POINT, 'client.connected'). -define(MAX_AUTO_SUBSCRIBE, 20). @@ -80,8 +82,18 @@ format(Rule = #{topic := Topic}) when is_map(Rule) -> }. update_(Topics) when length(Topics) =< ?MAX_AUTO_SUBSCRIBE -> - {ok, _} = emqx:update_config([auto_subscribe, topics], Topics), - update_hook(); + case emqx_conf:update([auto_subscribe, topics], + Topics, + #{rawconf_with_defaults => true, override_to => cluster}) of + {ok, #{config := NewTopics}} -> + ok = update_hook(), + {ok, NewTopics}; + {error, Reason} -> + ?LOG(error, "Auto Subscribe update config failed: ~0p", [Reason]), + {error, Reason} + end; + % {ok, _} = emqx:update_config([auto_subscribe, topics], Topics), + % update_hook(); update_(_Topics) -> {error, quota_exceeded}. diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl index d1207544a..cb5372d5d 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl @@ -22,6 +22,7 @@ -export([auto_subscribe/2]). +-define(INTERNAL_ERROR, 'INTERNAL_ERROR'). -define(EXCEED_LIMIT, 'EXCEED_LIMIT'). -define(BAD_REQUEST, 'BAD_REQUEST'). @@ -90,6 +91,9 @@ auto_subscribe(put, #{body := Params}) -> Message = list_to_binary(io_lib:format("Max auto subscribe topic count is ~p", [emqx_auto_subscribe:max_limit()])), {409, #{code => ?EXCEED_LIMIT, message => Message}}; - ok -> - {200, emqx_auto_subscribe:list()} + {error, Reason} -> + Message = list_to_binary(io_lib:format("Update config failed ~p", [Reason])), + {500, #{code => ?INTERNAL_ERROR, message => Message}}; + {ok, NewTopics} -> + {200, NewTopics} end. From 23cf74d82976f79d74a3c7abd9d83b8c48b68af3 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 30 Dec 2021 16:43:09 +0800 Subject: [PATCH 64/99] fix(delayed): update config in cluster --- apps/emqx_modules/src/emqx_delayed.erl | 2 +- apps/emqx_modules/src/emqx_delayed_api.erl | 73 +++++++++++----------- 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 36569d52e..73a21d3e6 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -187,7 +187,7 @@ delete_delayed_message(Id0) -> mria:dirty_delete(?TAB, {Timestamp, Id}) end. update_config(Config) -> - {ok, _} = emqx:update_config([delayed], Config). + emqx_conf:update([delayed], Config, #{rawconf_with_defaults => true, override_to => cluster}). %%-------------------------------------------------------------------- %% gen_server callback diff --git a/apps/emqx_modules/src/emqx_delayed_api.erl b/apps/emqx_modules/src/emqx_delayed_api.erl index 9199d7b2c..e582d9189 100644 --- a/apps/emqx_modules/src/emqx_delayed_api.erl +++ b/apps/emqx_modules/src/emqx_delayed_api.erl @@ -25,12 +25,14 @@ -define(MAX_PAYLOAD_LENGTH, 2048). -define(PAYLOAD_TOO_LARGE, 'PAYLOAD_TOO_LARGE'). --export([status/2 - , delayed_messages/2 - , delayed_message/2 -]). +-export([ status/2 + , delayed_messages/2 + , delayed_message/2 + ]). --export([paths/0, fields/1, schema/1]). +-export([ paths/0 + , fields/1 + , schema/1]). %% for rpc -export([update_config_/1]). @@ -40,6 +42,7 @@ -define(ALREADY_ENABLED, 'ALREADY_ENABLED'). -define(ALREADY_DISABLED, 'ALREADY_DISABLED'). +-define(INTERNAL_ERROR, 'INTERNAL_ERROR'). -define(BAD_REQUEST, 'BAD_REQUEST'). -define(MESSAGE_ID_NOT_FOUND, 'MESSAGE_ID_NOT_FOUND'). @@ -49,7 +52,11 @@ api_spec() -> emqx_dashboard_swagger:spec(?MODULE). -paths() -> ["/mqtt/delayed", "/mqtt/delayed/messages", "/mqtt/delayed/messages/:msgid"]. +paths() -> + [ "/mqtt/delayed" + , "/mqtt/delayed/messages" + , "/mqtt/delayed/messages/:msgid" + ]. schema("/mqtt/delayed") -> #{ @@ -189,8 +196,7 @@ get_status() -> update_config(Config) -> case generate_config(Config) of {ok, Config} -> - update_config_(Config), - {200, get_status()}; + update_config_(Config); {error, {Code, Message}} -> {400, #{code => Code, message => Message}} end. @@ -215,29 +221,28 @@ generate_max_delayed_messages(Config) -> {ok, Config}. update_config_(Config) -> - lists:foreach(fun(Node) -> - update_config_(Node, Config) - end, mria_mnesia:running_nodes()). - -update_config_(Node, Config) when Node =:= node() -> - _ = emqx_delayed:update_config(Config), - case maps:get(<<"enable">>, Config, undefined) of - undefined -> - ignore; - true -> - emqx_delayed:enable(); - false -> - emqx_delayed:disable() - end, - case maps:get(<<"max_delayed_messages">>, Config, undefined) of - undefined -> - ignore; - Max -> - ok = emqx_delayed:set_max_delayed_messages(Max) - end; - -update_config_(Node, Config) -> - rpc_call(Node, ?MODULE, ?FUNCTION_NAME, [Node, Config]). + case emqx_delayed:update_config(Config) of + {ok, #{config := NewDelayed}} -> + case maps:get(<<"enable">>, Config, undefined) of + undefined -> + ignore; + true -> + emqx_delayed:enable(); + false -> + emqx_delayed:disable() + end, + case maps:get(<<"max_delayed_messages">>, Config, undefined) of + undefined -> + ignore; + Max -> + ok = emqx_delayed:set_max_delayed_messages(Max) + end, + {200, NewDelayed}; + {error, Reason} -> + Message = list_to_binary( + io_lib:format("Update delayed message config failed ~p", [Reason])), + {500, ?INTERNAL_ERROR, Message} + end. generate_http_code_map(id_schema_error, Id) -> #{code => ?MESSAGE_ID_SCHEMA_ERROR, message => @@ -245,9 +250,3 @@ generate_http_code_map(id_schema_error, Id) -> generate_http_code_map(not_found, Id) -> #{code => ?MESSAGE_ID_NOT_FOUND, message => iolist_to_binary(io_lib:format("Message ID ~p not found", [Id]))}. - -rpc_call(Node, Module, Fun, Args) -> - case rpc:call(Node, Module, Fun, Args) of - {badrpc, Reason} -> {error, Reason}; - Result -> Result - end. From 6c52fb4806e92091186b79ce76fd09ad594e5f6d Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 30 Dec 2021 16:50:19 +0800 Subject: [PATCH 65/99] fix: code format --- apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl index 564772fe4..0183726da 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl @@ -16,8 +16,6 @@ -module(emqx_auto_subscribe). --include_lib("emqx/include/logger.hrl"). - -define(HOOK_POINT, 'client.connected'). -define(MAX_AUTO_SUBSCRIBE, 20). @@ -89,11 +87,8 @@ update_(Topics) when length(Topics) =< ?MAX_AUTO_SUBSCRIBE -> ok = update_hook(), {ok, NewTopics}; {error, Reason} -> - ?LOG(error, "Auto Subscribe update config failed: ~0p", [Reason]), {error, Reason} end; - % {ok, _} = emqx:update_config([auto_subscribe, topics], Topics), - % update_hook(); update_(_Topics) -> {error, quota_exceeded}. From f0330d9334737f06d23bb3b338e8af8248ddaf29 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 30 Dec 2021 17:36:36 +0800 Subject: [PATCH 66/99] fix(event_message): update config in cluster --- apps/emqx_modules/src/emqx_delayed_api.erl | 2 +- apps/emqx_modules/src/emqx_event_message.erl | 11 +++++++++-- apps/emqx_modules/src/emqx_event_message_api.erl | 9 +++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/apps/emqx_modules/src/emqx_delayed_api.erl b/apps/emqx_modules/src/emqx_delayed_api.erl index e582d9189..66d4a0fdc 100644 --- a/apps/emqx_modules/src/emqx_delayed_api.erl +++ b/apps/emqx_modules/src/emqx_delayed_api.erl @@ -240,7 +240,7 @@ update_config_(Config) -> {200, NewDelayed}; {error, Reason} -> Message = list_to_binary( - io_lib:format("Update delayed message config failed ~p", [Reason])), + io_lib:format("Update config failed ~p", [Reason])), {500, ?INTERNAL_ERROR, Message} end. diff --git a/apps/emqx_modules/src/emqx_event_message.erl b/apps/emqx_modules/src/emqx_event_message.erl index ccdb75ccb..3918e4dfc 100644 --- a/apps/emqx_modules/src/emqx_event_message.erl +++ b/apps/emqx_modules/src/emqx_event_message.erl @@ -44,8 +44,15 @@ list() -> update(Params) -> disable(), - {ok, _} = emqx:update_config([event_message], Params), - enable(). + case emqx_conf:update([event_message], + Params, + #{rawconf_with_defaults => true, override_to => cluster}) of + {ok, #{config := NewEventMessage}} -> + enable(), + {ok, NewEventMessage}; + {error, Reason} -> + {error, Reason} + end. enable() -> lists:foreach(fun({_Topic, false}) -> ok; diff --git a/apps/emqx_modules/src/emqx_event_message_api.erl b/apps/emqx_modules/src/emqx_event_message_api.erl index 80e5825d1..e27311e15 100644 --- a/apps/emqx_modules/src/emqx_event_message_api.erl +++ b/apps/emqx_modules/src/emqx_event_message_api.erl @@ -53,5 +53,10 @@ event_message(get, _Params) -> {200, emqx_event_message:list()}; event_message(put, #{body := Body}) -> - _ = emqx_event_message:update(Body), - {200, emqx_event_message:list()}. + case emqx_event_message:update(Body) of + {ok, NewConfig} -> + {200, NewConfig}; + {error, Reason} -> + Message = list_to_binary(io_lib:format("Update config failed ~p", [Reason])), + {500, 'INTERNAL_ERROR', Message} + end. From a630044688d53ff30264ba788bf05e5b967842d7 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 30 Dec 2021 18:35:20 +0800 Subject: [PATCH 67/99] fix: update result by row_config --- apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl | 2 +- apps/emqx_modules/src/emqx_delayed_api.erl | 2 +- apps/emqx_modules/src/emqx_event_message.erl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl index 0183726da..81b0d70a4 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl @@ -83,7 +83,7 @@ update_(Topics) when length(Topics) =< ?MAX_AUTO_SUBSCRIBE -> case emqx_conf:update([auto_subscribe, topics], Topics, #{rawconf_with_defaults => true, override_to => cluster}) of - {ok, #{config := NewTopics}} -> + {ok, #{raw_config := NewTopics}} -> ok = update_hook(), {ok, NewTopics}; {error, Reason} -> diff --git a/apps/emqx_modules/src/emqx_delayed_api.erl b/apps/emqx_modules/src/emqx_delayed_api.erl index 66d4a0fdc..5caad8aa1 100644 --- a/apps/emqx_modules/src/emqx_delayed_api.erl +++ b/apps/emqx_modules/src/emqx_delayed_api.erl @@ -222,7 +222,7 @@ generate_max_delayed_messages(Config) -> update_config_(Config) -> case emqx_delayed:update_config(Config) of - {ok, #{config := NewDelayed}} -> + {ok, #{raw_config := NewDelayed}} -> case maps:get(<<"enable">>, Config, undefined) of undefined -> ignore; diff --git a/apps/emqx_modules/src/emqx_event_message.erl b/apps/emqx_modules/src/emqx_event_message.erl index 3918e4dfc..3af57a38d 100644 --- a/apps/emqx_modules/src/emqx_event_message.erl +++ b/apps/emqx_modules/src/emqx_event_message.erl @@ -47,7 +47,7 @@ update(Params) -> case emqx_conf:update([event_message], Params, #{rawconf_with_defaults => true, override_to => cluster}) of - {ok, #{config := NewEventMessage}} -> + {ok, #{raw_config := NewEventMessage}} -> enable(), {ok, NewEventMessage}; {error, Reason} -> From 173ae4653803860a8aa1ebbf9943ca62c2057997 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 31 Dec 2021 10:04:06 +0800 Subject: [PATCH 68/99] fix(auto_subscribe): bad test suite --- .../emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl index 0e5022533..ab8f9a408 100644 --- a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl +++ b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl @@ -85,7 +85,7 @@ init_per_suite(Config) -> } ] }">>), - emqx_common_test_helpers:start_apps([emqx_dashboard, ?APP], fun set_special_configs/1), + emqx_common_test_helpers:start_apps([emqx_dashboard, emqx_conf, ?APP], fun set_special_configs/1), Config. set_special_configs(emqx_dashboard) -> @@ -113,15 +113,17 @@ topic_config(T) -> end_per_suite(_) -> application:unload(emqx_management), + application:unload(emqx_conf), application:unload(?APP), meck:unload(emqx_resource), meck:unload(emqx_schema), emqx_common_test_helpers:stop_apps([emqx_dashboard, ?APP]). t_auto_subscribe(_) -> + emqx_auto_subscribe:update([#{<<"topic">> => Topic} || Topic <- ?TOPICS]), {ok, Client} = emqtt:start_link(#{username => ?CLIENT_USERNAME, clientid => ?CLIENT_ID}), {ok, _} = emqtt:connect(Client), - timer:sleep(100), + timer:sleep(200), ?assertEqual(check_subs(length(?TOPICS)), ok), emqtt:disconnect(Client), ok. @@ -148,6 +150,7 @@ t_update(_) -> check_subs(Count) -> Subs = ets:tab2list(emqx_suboption), + ct:pal("---> ~p ~p ~n", [Subs, Count]), ?assert(length(Subs) >= Count), check_subs((Subs), ?ENSURE_TOPICS). From 3a38e8d68dda45a5f711c382fef571e8762ec512 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 31 Dec 2021 11:00:55 +0800 Subject: [PATCH 69/99] fix(statsd): update config in cluster --- apps/emqx_statsd/src/emqx_statsd.erl | 2 +- apps/emqx_statsd/src/emqx_statsd_api.erl | 26 +++++++++++++++--------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/apps/emqx_statsd/src/emqx_statsd.erl b/apps/emqx_statsd/src/emqx_statsd.erl index 892731a6c..5f88e5bdd 100644 --- a/apps/emqx_statsd/src/emqx_statsd.erl +++ b/apps/emqx_statsd/src/emqx_statsd.erl @@ -94,7 +94,7 @@ terminate(_Reason, #state{estatsd_pid = Pid}) -> ok. %%------------------------------------------------------------------------------ -%% Internale function +%% Internal function %%------------------------------------------------------------------------------ trans_metrics_name(Name) -> Name0 = atom_to_binary(Name, utf8), diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl index 97e803f5b..2278fa492 100644 --- a/apps/emqx_statsd/src/emqx_statsd_api.erl +++ b/apps/emqx_statsd/src/emqx_statsd_api.erl @@ -55,13 +55,19 @@ statsd(get, _Params) -> {200, emqx:get_raw_config([<<"statsd">>], #{})}; statsd(put, #{body := Body}) -> - {ok, Config} = emqx:update_config([statsd], Body), - case maps:get(<<"enable">>, Body) of - true -> - _ = emqx_statsd_sup:stop_child(?APP), - emqx_statsd_sup:start_child(?APP, maps:get(config, Config)); - false -> - _ = emqx_statsd_sup:stop_child(?APP), - ok - end, - {200, emqx:get_raw_config([<<"statsd">>], #{})}. + case emqx:update_config([statsd], + Body, + #{rawconf_with_defaults => true, override_to => cluster}) of + {ok, #{raw_config := NewConfig, config := Config}} -> + case maps:get(<<"enable">>, Body) of + true -> + _ = emqx_statsd_sup:stop_child(?APP), + emqx_statsd_sup:start_child(?APP, maps:get(config, Config)); + false -> + _ = emqx_statsd_sup:stop_child(?APP) + end, + {200, NewConfig}; + {error, Reason} -> + Message = list_to_binary(io_lib:format("Update config failed ~p", [Reason])), + {500, 'INTERNAL_ERROR', Message} + end. From ebbb473d7a776c55cc115982734536415c01819f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 29 Dec 2021 10:10:48 +0800 Subject: [PATCH 70/99] refactor(topic-metrics): cluster supported --- apps/emqx_modules/src/emqx_modules_app.erl | 6 +- apps/emqx_modules/src/emqx_modules_conf.erl | 131 ++++++++++++ apps/emqx_modules/src/emqx_topic_metrics.erl | 15 -- .../src/emqx_topic_metrics_api.erl | 202 ++++++++++++------ .../test/emqx_modules_conf_SUITE.erl | 51 +++++ 5 files changed, 324 insertions(+), 81 deletions(-) create mode 100644 apps/emqx_modules/src/emqx_modules_conf.erl create mode 100644 apps/emqx_modules/test/emqx_modules_conf_SUITE.erl diff --git a/apps/emqx_modules/src/emqx_modules_app.erl b/apps/emqx_modules/src/emqx_modules_app.erl index 55c882f94..4d49f22c8 100644 --- a/apps/emqx_modules/src/emqx_modules_app.erl +++ b/apps/emqx_modules/src/emqx_modules_app.erl @@ -38,7 +38,8 @@ maybe_enable_modules() -> emqx_event_message:enable(), emqx_conf_cli:load(), ok = emqx_rewrite:enable(), - emqx_topic_metrics:enable(). + emqx_topic_metrics:enable(), + emqx_modules_conf:load(). maybe_disable_modules() -> emqx_conf:get([delayed, enable], true) andalso emqx_delayed:disable(), @@ -47,4 +48,5 @@ maybe_disable_modules() -> emqx_event_message:disable(), emqx_rewrite:disable(), emqx_conf_cli:unload(), - emqx_topic_metrics:disable(). + emqx_topic_metrics:disable(), + emqx_modules_conf:unload(). diff --git a/apps/emqx_modules/src/emqx_modules_conf.erl b/apps/emqx_modules/src/emqx_modules_conf.erl new file mode 100644 index 000000000..a757e0878 --- /dev/null +++ b/apps/emqx_modules/src/emqx_modules_conf.erl @@ -0,0 +1,131 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The emqx-modules configration interoperable interfaces +-module(emqx_modules_conf). + +-behaviour(emqx_config_handler). + +%% Load/Unload +-export([ load/0 + , unload/0 + ]). + +%% topci-metrics +-export([ topic_metrics/0 + , add_topic_metrics/1 + , remove_topic_metrics/1 + ]). + +%% config handlers +-export([ pre_config_update/3 + , post_config_update/5 + ]). + +%%-------------------------------------------------------------------- +%% Load/Unload + +-spec load() -> ok. +load() -> + emqx_conf:add_handler([topic_metrics], ?MODULE). + +-spec unload() -> ok. +unload() -> + emqx_conf:remove_handler([topic_metrics]). + +%%-------------------------------------------------------------------- +%% Topic-Metrics + +-spec topic_metrics() -> [emqx_types:topic()]. +topic_metrics() -> + lists:map( + fun(#{topic := Topic}) -> Topic end, + emqx:get_config([topic_metrics]) + ). + +-spec add_topic_metrics(emqx_types:topic()) + -> {ok, emqx_types:topic()} + | {error, term()}. +add_topic_metrics(Topic) -> + case cfg_update(topic_metrics, ?FUNCTION_NAME, Topic) of + {ok, _} -> {ok, Topic}; + {error, Reason} -> {error, Reason} + end. + +-spec remove_topic_metrics(emqx_types:topic()) + -> ok + | {error, term()}. +remove_topic_metrics(Topic) -> + case cfg_update(topic_metrics, ?FUNCTION_NAME, Topic) of + {ok, _} -> ok; + {error, Reason} -> {error, Reason} + end. + +cfg_update(topic_metrics, Action, Params) -> + res(emqx_conf:update( + [topic_metrics], + {Action, Params}, + #{override_to => cluster})). + +res({ok, Result}) -> {ok, Result}; +res({error, {pre_config_update,?MODULE,Reason}}) -> {error, Reason}; +res({error, {post_config_update,?MODULE,Reason}}) -> {error, Reason}; +res({error, Reason}) -> {error, Reason}. + +%%-------------------------------------------------------------------- +%% Config Handler +%%-------------------------------------------------------------------- + +-spec pre_config_update(list(atom()), + emqx_config:update_request(), + emqx_config:raw_config()) -> + {ok, emqx_config:update_request()} | {error, term()}. +pre_config_update(_, {add_topic_metrics, Topic0}, RawConf) -> + Topic = #{<<"topic">> => Topic0}, + case lists:member(Topic, RawConf) of + true -> + {error, already_exist}; + _ -> + {ok, RawConf ++ [Topic]} + end; +pre_config_update(_, {remove_topic_metrics, Topic0}, RawConf) -> + Topic = #{<<"topic">> => Topic0}, + case lists:member(Topic, RawConf) of + true -> + {ok, RawConf -- [Topic]}; + _ -> + {error, not_found} + end. + +-spec post_config_update(list(atom()), + emqx_config:update_request(), + emqx_config:config(), + emqx_config:config(), emqx_config:app_envs()) + -> ok | {ok, Result::any()} | {error, Reason::term()}. + +post_config_update(_, {add_topic_metrics, Topic}, + _NewConfig, _OldConfig, _AppEnvs) -> + case emqx_topic_metrics:register(Topic) of + ok -> ok; + {error, Reason} -> {error, Reason} + end; + +post_config_update(_, {remove_topic_metrics, Topic}, + _NewConfig, _OldConfig, _AppEnvs) -> + case emqx_topic_metrics:deregister(Topic) of + ok -> ok; + {error, Reason} -> {error, Reason} + end. diff --git a/apps/emqx_modules/src/emqx_topic_metrics.erl b/apps/emqx_modules/src/emqx_topic_metrics.erl index 7ca14b921..e4a746139 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics.erl @@ -220,7 +220,6 @@ handle_call({register, Topic}, _From, State = #state{speeds = Speeds}) -> handle_call({deregister, all}, _From, State) -> true = ets:delete_all_objects(?TAB), - update_config([]), {reply, ok, State#state{speeds = #{}}}; handle_call({deregister, Topic}, _From, State = #state{speeds = Speeds}) -> @@ -232,7 +231,6 @@ handle_call({deregister, Topic}, _From, State = #state{speeds = Speeds}) -> NSpeeds = lists:foldl(fun(Metric, Acc) -> maps:remove({Topic, Metric}, Acc) end, Speeds, ?TOPIC_METRICS), - remove_topic_config(Topic), {reply, ok, State#state{speeds = NSpeeds}} end; @@ -316,7 +314,6 @@ do_register(Topic, Speeds) -> NSpeeds = lists:foldl(fun(Metric, Acc) -> maps:put({Topic, Metric}, #speed{}, Acc) end, Speeds, ?TOPIC_METRICS), - add_topic_config(Topic), {ok, NSpeeds}; {true, true} -> {error, bad_topic}; @@ -351,18 +348,6 @@ format({Topic, Data}) -> TopicMetrics#{reset_time => ResetTime} end. -remove_topic_config(Topic) when is_binary(Topic) -> - Topics = emqx_config:get_raw([<<"topic_metrics">>], []) -- [#{<<"topic">> => Topic}], - update_config(Topics). - -add_topic_config(Topic) when is_binary(Topic) -> - Topics = emqx_config:get_raw([<<"topic_metrics">>], []) ++ [#{<<"topic">> => Topic}], - update_config(lists:usort(Topics)). - -update_config(Topics) when is_list(Topics) -> - {ok, _} = emqx:update_config([topic_metrics], Topics), - ok. - try_inc(Topic, Metric) -> _ = inc(Topic, Metric), ok. diff --git a/apps/emqx_modules/src/emqx_topic_metrics_api.erl b/apps/emqx_modules/src/emqx_topic_metrics_api.erl index e8be39c47..a45a8fd33 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics_api.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics_api.erl @@ -73,6 +73,7 @@ properties() -> topic_metrics_api() -> MetaData = #{ + %% Get all nodes metrics and accumulate all of these get => #{ description => <<"List topic metrics">>, responses => #{ @@ -133,87 +134,160 @@ topic_param() -> }. %%-------------------------------------------------------------------- -%% api callback +%% HTTP Callbacks +%%-------------------------------------------------------------------- + topic_metrics(get, _) -> - list_metrics(); + case cluster_accumulation_metrics() of + {error, Reason} -> + {500, Reason}; + {ok, Metrics} -> + {200, Metrics} + end; + topic_metrics(put, #{body := #{<<"topic">> := Topic, <<"action">> := <<"reset">>}}) -> - reset(Topic); + case reset(Topic) of + ok -> {200}; + {error, Reason} -> reason2httpresp(Reason) + end; topic_metrics(put, #{body := #{<<"action">> := <<"reset">>}}) -> - reset(); + case reset() of + ok -> {200}; + {error, Reason} -> reason2httpresp(Reason) + end; + topic_metrics(post, #{body := #{<<"topic">> := <<>>}}) -> {400, 'BAD_REQUEST', <<"Topic can not be empty">>}; topic_metrics(post, #{body := #{<<"topic">> := Topic}}) -> - register(Topic). - -operate_topic_metrics(Method, #{bindings := #{topic := Topic0}}) -> - Topic = decode_topic(Topic0), - case Method of - get -> - get_metrics(Topic); - put -> - register(Topic); - delete -> - deregister(Topic) + case emqx_modules_conf:add_topic_metrics(Topic) of + {ok, Topic} -> + {200}; + {error, Reason} -> + reason2httpresp(Reason) end. -decode_topic(Topic) -> - uri_string:percent_decode(Topic). +operate_topic_metrics(get, #{bindings := #{topic := Topic0}}) -> + case cluster_accumulation_metrics(emqx_http_lib:uri_decode(Topic0)) of + {ok, Metrics} -> + {200, Metrics}; + {error, Reason} -> + reason2httpresp(Reason) + end; + +operate_topic_metrics(delete, #{bindings := #{topic := Topic0}}) -> + case emqx_modules_conf:remove_topic_metrics(emqx_http_lib:uri_decode(Topic0)) of + ok -> {200}; + {error, Reason} -> + reason2httpresp(Reason) + end. %%-------------------------------------------------------------------- -%% api apply -list_metrics() -> - {200, emqx_topic_metrics:metrics()}. +%% Internal funcs +%%-------------------------------------------------------------------- -register(Topic) -> - case emqx_topic_metrics:register(Topic) of - {error, quota_exceeded} -> - Message = list_to_binary(io_lib:format("Max topic metrics count is ~p", - [emqx_topic_metrics:max_limit()])), - {409, #{code => ?EXCEED_LIMIT, message => Message}}; - {error, bad_topic} -> - Message = list_to_binary(io_lib:format("Bad Topic, topic cannot have wildcard ~p", - [Topic])), - {400, #{code => ?BAD_TOPIC, message => Message}}; - {error, {quota_exceeded, bad_topic}} -> - Message = list_to_binary( - io_lib:format( - "Max topic metrics count is ~p, and topic cannot have wildcard ~p", - [emqx_topic_metrics:max_limit(), Topic])), - {400, #{code => ?BAD_REQUEST, message => Message}}; - {error, already_existed} -> - Message = list_to_binary(io_lib:format("Topic ~p already registered", [Topic])), - {400, #{code => ?BAD_TOPIC, message => Message}}; - ok -> - {200} +cluster_accumulation_metrics() -> + case multicall(emqx_topic_metrics, metrics, []) of + {SuccResList, []} -> + {ok, accumulate_nodes_metrics(SuccResList)}; + {_, FailedNodes} -> + {error, {badrpc, FailedNodes}} end. -deregister(Topic) -> - case emqx_topic_metrics:deregister(Topic) of - {error, topic_not_found} -> - Message = list_to_binary(io_lib:format("Topic ~p not found", [Topic])), - {404, #{code => ?ERROR_TOPIC, message => Message}}; - ok -> - {200} +cluster_accumulation_metrics(Topic) -> + case multicall(emqx_topic_metrics, metrics, [Topic]) of + {SuccResList, []} -> + case lists:filter(fun({error, _}) -> false; (_) -> true + end, SuccResList) of + [] -> {error, topic_not_found}; + TopicMetrics -> + NTopicMetrics = [ [T] || T <- TopicMetrics], + [AccMetrics] = accumulate_nodes_metrics(NTopicMetrics), + {ok, AccMetrics} + end; + {_, FailedNodes} -> + {error, {badrpc, FailedNodes}} end. -get_metrics(Topic) -> - case emqx_topic_metrics:metrics(Topic) of - {error, topic_not_found} -> - Message = list_to_binary(io_lib:format("Topic ~p not found", [Topic])), - {404, #{code => ?ERROR_TOPIC, message => Message}}; - Metrics -> - {200, Metrics} - end. +accumulate_nodes_metrics(NodesTopicMetrics) -> + AccMap = lists:foldl(fun(TopicMetrics, ExAcc) -> + MetricsMap = lists:foldl( + fun(#{topic := Topic, + metrics := Metrics, + create_time := CreateTime}, Acc) -> + Acc#{Topic => {Metrics, CreateTime}} + end, #{}, TopicMetrics), + accumulate_metrics(MetricsMap, ExAcc) + end, #{}, NodesTopicMetrics), + maps:fold(fun(Topic, {Metrics, CreateTime1}, Acc1) -> + [#{topic => Topic, + metrics => Metrics, + create_time => CreateTime1} | Acc1] + end, [], AccMap). + +%% @doc TopicMetricsIn :: #{<<"topic">> := {Metrics, CreateTime}} +accumulate_metrics(TopicMetricsIn, TopicMetricsAcc) -> + Topics = maps:keys(TopicMetricsIn), + lists:foldl(fun(Topic, Acc) -> + {Metrics, CreateTime} = maps:get(Topic, TopicMetricsIn), + NMetrics = do_accumulation_metrics( + Metrics, + maps:get(Topic, TopicMetricsAcc, undefined) + ), + maps:put(Topic, {NMetrics, CreateTime}, Acc) + end, #{}, Topics). + +%% @doc MetricsIn :: #{'messages.dropped.rate' :: integer(), ...} +do_accumulation_metrics(MetricsIn, undefined) -> MetricsIn; +do_accumulation_metrics(MetricsIn, MetricsAcc) -> + Keys = maps:keys(MetricsIn), + lists:foldl(fun(Key, Acc) -> + InVal = maps:get(Key, MetricsIn), + NVal = InVal + maps:get(Key, MetricsAcc, 0), + maps:put(Key, NVal, Acc) + end, #{}, Keys). reset() -> - ok = emqx_topic_metrics:reset(), - {200}. + _ = multicall(emqx_topic_metrics, reset, []), + ok. reset(Topic) -> - case emqx_topic_metrics:reset(Topic) of - {error, topic_not_found} -> - Message = list_to_binary(io_lib:format("Topic ~p not found", [Topic])), - {404, #{code => ?ERROR_TOPIC, message => Message}}; - ok -> - {200} + case multicall(emqx_topic_metrics, reset, [Topic]) of + {SuccResList, []} -> + case lists:filter(fun({error, _}) -> true; (_) -> false + end, SuccResList) of + [{error, Reason} | _] -> + {error, Reason}; + [] -> + ok + end end. + +%%-------------------------------------------------------------------- +%% utils + +multicall(M, F, A) -> + emqx_rpc:multicall(mria_mnesia:running_nodes(), M, F, A). + +reason2httpresp(quota_exceeded) -> + Msg = list_to_binary( + io_lib:format("Max topic metrics count is ~p", + [emqx_topic_metrics:max_limit()])), + {409, #{code => ?EXCEED_LIMIT, message => Msg}}; +reason2httpresp(bad_topic) -> + Msg = <<"Bad Topic, topic cannot have wildcard">>, + {400, #{code => ?BAD_TOPIC, message => Msg}}; +reason2httpresp({quota_exceeded, bad_topic}) -> + Msg = list_to_binary( + io_lib:format( + "Max topic metrics count is ~p, and topic cannot have wildcard", + [emqx_topic_metrics:max_limit()])), + {400, #{code => ?BAD_REQUEST, message => Msg}}; +reason2httpresp(already_existed) -> + Msg = <<"Topic already registered">>, + {400, #{code => ?BAD_TOPIC, message => Msg}}; +reason2httpresp(not_found) -> + Msg = <<"Topic not found">>, + {404, #{code => ?ERROR_TOPIC, message => Msg}}; +reason2httpresp(topic_not_found) -> + Msg = <<"Topic not found">>, + {404, #{code => ?ERROR_TOPIC, message => Msg}}. diff --git a/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl b/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl new file mode 100644 index 000000000..95ebb5711 --- /dev/null +++ b/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl @@ -0,0 +1,51 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_modules_conf_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Conf) -> + emqx_config:init_load(emqx_modules_schema, <<"gateway {}">>), + emqx_common_test_helpers:start_apps([emqx_conf, emqx_modules]), + Conf. + +end_per_suite(_Conf) -> + emqx_common_test_helpers:stop_apps([emqx_modules, emqx_conf]). + +init_per_testcase(_CaseName, Conf) -> + Conf. + +%%-------------------------------------------------------------------- +%% Cases +%%-------------------------------------------------------------------- + +t_topic_metrics_list(_) -> + ok. + +t_topic_metrics_add_remove(_) -> + ok. + From 6e6643f974cd848027fb0119a1e5f281b2ecdbc0 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Dec 2021 18:42:58 +0800 Subject: [PATCH 71/99] fix: fix dialyzer warnings --- apps/emqx_modules/src/emqx_modules_conf.erl | 6 +++--- .../emqx_modules/src/emqx_topic_metrics_api.erl | 17 +++++++---------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/apps/emqx_modules/src/emqx_modules_conf.erl b/apps/emqx_modules/src/emqx_modules_conf.erl index a757e0878..386f269f0 100644 --- a/apps/emqx_modules/src/emqx_modules_conf.erl +++ b/apps/emqx_modules/src/emqx_modules_conf.erl @@ -81,8 +81,8 @@ cfg_update(topic_metrics, Action, Params) -> #{override_to => cluster})). res({ok, Result}) -> {ok, Result}; -res({error, {pre_config_update,?MODULE,Reason}}) -> {error, Reason}; -res({error, {post_config_update,?MODULE,Reason}}) -> {error, Reason}; +res({error, {pre_config_update, ?MODULE, Reason}}) -> {error, Reason}; +res({error, {post_config_update, ?MODULE, Reason}}) -> {error, Reason}; res({error, Reason}) -> {error, Reason}. %%-------------------------------------------------------------------- @@ -97,7 +97,7 @@ pre_config_update(_, {add_topic_metrics, Topic0}, RawConf) -> Topic = #{<<"topic">> => Topic0}, case lists:member(Topic, RawConf) of true -> - {error, already_exist}; + {error, already_existed}; _ -> {ok, RawConf ++ [Topic]} end; diff --git a/apps/emqx_modules/src/emqx_topic_metrics_api.erl b/apps/emqx_modules/src/emqx_topic_metrics_api.erl index a45a8fd33..1ba76579b 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics_api.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics_api.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- -%% TODO: refactor uri path + -module(emqx_topic_metrics_api). -behaviour(minirest_api). @@ -151,10 +151,8 @@ topic_metrics(put, #{body := #{<<"topic">> := Topic, <<"action">> := <<"reset">> {error, Reason} -> reason2httpresp(Reason) end; topic_metrics(put, #{body := #{<<"action">> := <<"reset">>}}) -> - case reset() of - ok -> {200}; - {error, Reason} -> reason2httpresp(Reason) - end; + reset(), + {200}; topic_metrics(post, #{body := #{<<"topic">> := <<>>}}) -> {400, 'BAD_REQUEST', <<"Topic can not be empty">>}; @@ -177,8 +175,7 @@ operate_topic_metrics(get, #{bindings := #{topic := Topic0}}) -> operate_topic_metrics(delete, #{bindings := #{topic := Topic0}}) -> case emqx_modules_conf:remove_topic_metrics(emqx_http_lib:uri_decode(Topic0)) of ok -> {200}; - {error, Reason} -> - reason2httpresp(Reason) + {error, Reason} -> reason2httpresp(Reason) end. %%-------------------------------------------------------------------- @@ -285,9 +282,9 @@ reason2httpresp({quota_exceeded, bad_topic}) -> reason2httpresp(already_existed) -> Msg = <<"Topic already registered">>, {400, #{code => ?BAD_TOPIC, message => Msg}}; -reason2httpresp(not_found) -> - Msg = <<"Topic not found">>, - {404, #{code => ?ERROR_TOPIC, message => Msg}}; reason2httpresp(topic_not_found) -> + Msg = <<"Topic not found">>, + {404, #{code => ?ERROR_TOPIC, message => Msg}}; +reason2httpresp(not_found) -> Msg = <<"Topic not found">>, {404, #{code => ?ERROR_TOPIC, message => Msg}}. From c7693246febb4a5b6988547981deb8bacd8de23a Mon Sep 17 00:00:00 2001 From: EMQ-YangM Date: Tue, 28 Dec 2021 21:38:10 -0800 Subject: [PATCH 72/99] feat(emqx_resource): add health_ckeck process, it will periodically perform health checks, and print error logs and generate alarms when the checks fail. --- .../src/emqx_resource_health_check.erl | 43 +++++++++++++++++++ .../src/emqx_resource_health_check_sup.erl | 40 +++++++++++++++++ .../src/emqx_resource_instance.erl | 6 ++- apps/emqx_resource/src/emqx_resource_sup.erl | 7 ++- 4 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 apps/emqx_resource/src/emqx_resource_health_check.erl create mode 100644 apps/emqx_resource/src/emqx_resource_health_check_sup.erl diff --git a/apps/emqx_resource/src/emqx_resource_health_check.erl b/apps/emqx_resource/src/emqx_resource_health_check.erl new file mode 100644 index 000000000..50b236daa --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_health_check.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_resource_health_check). + +-export([child_spec/2]). + +-export([start_link/2]). + +-export([health_check/2]). + +child_spec(Name, Sleep) -> + #{id => {health_check, Name}, + start => {?MODULE, start_link, [Name, Sleep]}, + restart => transient, + shutdown => 5000, type => worker, modules => [?MODULE]}. + +start_link(Name, Sleep) -> + Pid = proc_lib:spawn_link(?MODULE, health_check, [Name, Sleep]), + {ok, Pid}. + +health_check(Name, SleepTime) -> + timer:sleep(SleepTime), + case emqx_resource:health_check(Name) of + ok -> + emqx_alarm:deactivate(Name); + {error, _} -> + emqx_alarm:activate(Name, #{name => Name}, + <>) + end, + health_check(Name, SleepTime). \ No newline at end of file diff --git a/apps/emqx_resource/src/emqx_resource_health_check_sup.erl b/apps/emqx_resource/src/emqx_resource_health_check_sup.erl new file mode 100644 index 000000000..571cd6338 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_health_check_sup.erl @@ -0,0 +1,40 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_resource_health_check_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1, + create_health_check_process/2, + delete_health_check_process/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + SupFlags = #{strategy => one_for_one, intensity => 10, period => 10}, + {ok, {SupFlags, []}}. + +create_health_check_process(Name, Sleep) -> + supervisor:start_child(emqx_resource_health_check_sup, + emqx_resource_health_check:child_spec(Name, Sleep)). + +delete_health_check_process(Name) -> + _ = supervisor:terminate_child(emqx_resource_health_check_sup, {health_check, Name}), + _ = supervisor:delete_child(emqx_resource_health_check_sup, {health_check, Name}), + ok. \ No newline at end of file diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index 745b8b684..af9c999cf 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -173,6 +173,8 @@ do_create(InstId, ResourceType, Config, Opts) -> %% this is the first time we do health check, this will update the %% status and then do ets:insert/2 _ = do_health_check(Res0#{state => ResourceState}), + HealthCheckInterval = maps:get(health_check_interval, Opts, 15000), + emqx_resource_health_check_sup:create_health_check_process(InstId, HealthCheckInterval), {ok, force_lookup(InstId)}; {error, Reason} when ForceCreate == true -> logger:error("start ~ts resource ~ts failed: ~p, " @@ -216,7 +218,9 @@ do_remove(Mod, InstId, ResourceState, ClearMetrics) -> case ClearMetrics of true -> ok = emqx_plugin_libs_metrics:clear_metrics(resource_metrics, InstId); false -> ok - end. + end, + _ = emqx_resource_health_check_sup:delete_health_check_process(InstId), + ok. do_restart(InstId) -> case lookup(InstId) of diff --git a/apps/emqx_resource/src/emqx_resource_sup.erl b/apps/emqx_resource/src/emqx_resource_sup.erl index 534777b69..b5655e301 100644 --- a/apps/emqx_resource/src/emqx_resource_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_sup.erl @@ -45,7 +45,12 @@ init([]) -> restart => transient, shutdown => 5000, type => worker, modules => [Mod]} end || Idx <- lists:seq(1, ?POOL_SIZE)], - {ok, {SupFlags, [Metrics | ResourceInsts]}}. + HealthCheck = + #{id => emqx_resource_health_check_sup, + start => {emqx_resource_health_check_sup, start_link, []}, + restart => transient, + shutdown => 5000, type => supervisor, modules => [emqx_resource_health_check_sup]}, + {ok, {SupFlags, [HealthCheck, Metrics | ResourceInsts]}}. %% internal functions ensure_pool(Pool, Type, Opts) -> From d18a2ab57c1dac3981114e931fcc67cefebeb06c Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 31 Dec 2021 12:07:53 +0800 Subject: [PATCH 73/99] fix(delayed): base64 encode twice --- apps/emqx_modules/src/emqx_delayed.erl | 2 +- apps/emqx_retainer/src/emqx_retainer_api.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 36569d52e..69e9ebb5b 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -155,7 +155,7 @@ format_delayed(#delayed_message{key = {ExpectTimeStamp, Id}, delayed = Delayed, }, case WithPayload of true -> - Result#{payload => base64:encode(Payload)}; + Result#{payload => Payload}; _ -> Result end. diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index b60ac5627..7739d60dc 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -84,7 +84,7 @@ with_topic_api() -> parameters => parameters(), responses => #{ <<"200">> => object_schema(message_props(), <<"List retained messages">>), - <<"404">> => error_schema(<<"Reatined Not Exists">>, ['NOT_FOUND']), + <<"404">> => error_schema(<<"Retained Not Exists">>, ['NOT_FOUND']), <<"405">> => schema(<<"NotAllowed">>) } }, From 6cde540fd1b1d4b4579b742ed971bf1dd503a62d Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 31 Dec 2021 14:17:50 +0800 Subject: [PATCH 74/99] fix(test): close app at end_per_suite --- apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl index ab8f9a408..92eb9a9ab 100644 --- a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl +++ b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl @@ -117,7 +117,7 @@ end_per_suite(_) -> application:unload(?APP), meck:unload(emqx_resource), meck:unload(emqx_schema), - emqx_common_test_helpers:stop_apps([emqx_dashboard, ?APP]). + emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_conf, ?APP]). t_auto_subscribe(_) -> emqx_auto_subscribe:update([#{<<"topic">> => Topic} || Topic <- ?TOPICS]), From e299d8d138ab8477638fd14317344ed117272bf4 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 31 Dec 2021 15:47:03 +0800 Subject: [PATCH 75/99] fix(rule): rules not triggered after the ingress mqtt bridge received some msg --- apps/emqx_bridge/src/emqx_bridge.erl | 23 +++++++++++++------ .../src/emqx_rule_runtime.erl | 11 +++++---- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index d46ce217e..1814a11fe 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -90,7 +90,14 @@ on_message_publish(Message = #message{topic = Topic, flags = Flags}) -> send_message(BridgeId, Message) -> {BridgeType, BridgeName} = parse_bridge_id(BridgeId), ResId = emqx_bridge:resource_id(BridgeType, BridgeName), - emqx_resource:query(ResId, {send_message, Message}). + case emqx:get_config([bridges, BridgeType, BridgeName], not_found) of + not_found -> + throw({bridge_not_found, BridgeId}); + #{enable := true} -> + emqx_resource:query(ResId, {send_message, Message}); + #{enable := false} -> + throw({bridge_stopped, BridgeId}) + end. config_key_path() -> [bridges]. @@ -279,6 +286,8 @@ get_matched_bridges(Topic) -> end, Acc0, Conf) end, [], Bridges). +get_matched_bridge_id(#{enable := false}, _Topic, _BType, _BName, Acc) -> + Acc; get_matched_bridge_id(#{local_topic := Filter}, Topic, BType, BName, Acc) -> case emqx_topic:match(Topic, Filter) of true -> [bridge_id(BType, BName) | Acc]; @@ -309,21 +318,21 @@ parse_confs(Type, Name, #{connector := ConnId, direction := Direction} = Conf) {Type, ConnName} -> ConnectorConfs = emqx:get_config([connectors, Type, ConnName]), make_resource_confs(Direction, ConnectorConfs, - maps:without([connector, direction], Conf), Name); + maps:without([connector, direction], Conf), Type, Name); {_ConnType, _ConnName} -> error({cannot_use_connector_with_different_type, ConnId}) end; -parse_confs(_Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf) +parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf) when is_map(ConnectorConfs) -> make_resource_confs(Direction, ConnectorConfs, - maps:without([connector, direction], Conf), Name). + maps:without([connector, direction], Conf), Type, Name). -make_resource_confs(ingress, ConnectorConfs, BridgeConf, Name) -> - BName = bin(Name), +make_resource_confs(ingress, ConnectorConfs, BridgeConf, Type, Name) -> + BName = bridge_id(Type, Name), ConnectorConfs#{ ingress => BridgeConf#{hookpoint => <<"$bridges/", BName/binary>>} }; -make_resource_confs(egress, ConnectorConfs, BridgeConf, _Name) -> +make_resource_confs(egress, ConnectorConfs, BridgeConf, _Type, _Name) -> ConnectorConfs#{ egress => BridgeConf }. diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 60a7cbaad..428a1ef8c 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -101,7 +101,7 @@ do_apply_rule(#{ true -> ok = emqx_plugin_libs_metrics:inc_matched(rule_metrics, RuleId), Collection2 = filter_collection(Input, InCase, DoEach, Collection), - {ok, [handle_output_list(Outputs, Coll, Input) || Coll <- Collection2]}; + {ok, [handle_output_list(RuleId, Outputs, Coll, Input) || Coll <- Collection2]}; false -> {error, nomatch} end; @@ -118,7 +118,7 @@ do_apply_rule(#{id := RuleId, {match_conditions_error, {_EXCLASS_,_EXCPTION_,_ST_}}) of true -> ok = emqx_plugin_libs_metrics:inc_matched(rule_metrics, RuleId), - {ok, handle_output_list(Outputs, Selected, Input)}; + {ok, handle_output_list(RuleId, Outputs, Selected, Input)}; false -> {error, nomatch} end. @@ -231,14 +231,15 @@ number(Bin) -> catch error:badarg -> binary_to_float(Bin) end. -handle_output_list(Outputs, Selected, Envs) -> - [handle_output(Out, Selected, Envs) || Out <- Outputs]. +handle_output_list(RuleId, Outputs, Selected, Envs) -> + [handle_output(RuleId, Out, Selected, Envs) || Out <- Outputs]. -handle_output(OutId, Selected, Envs) -> +handle_output(RuleId, OutId, Selected, Envs) -> try do_handle_output(OutId, Selected, Envs) catch Err:Reason:ST -> + ok = emqx_plugin_libs_metrics:inc_failed(rule_metrics, RuleId), ?SLOG(error, #{msg => "output_failed", output => OutId, exception => Err, From 657ecef67bec35b769bb59ec2087caaadd474d6a Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 31 Dec 2021 20:57:34 +0800 Subject: [PATCH 76/99] fix(resource): don't crash on resource stopped --- apps/emqx_bridge/src/emqx_bridge.erl | 5 ++++- apps/emqx_resource/include/emqx_resource.hrl | 4 +++- apps/emqx_resource/src/emqx_resource.erl | 16 +++++++++++----- .../emqx_resource/src/emqx_resource_instance.erl | 14 +++++++++++--- apps/emqx_resource/test/emqx_resource_SUITE.erl | 2 +- apps/emqx_rule_engine/src/emqx_rule_runtime.erl | 3 ++- 6 files changed, 32 insertions(+), 12 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 1814a11fe..3ae4e5820 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -94,7 +94,10 @@ send_message(BridgeId, Message) -> not_found -> throw({bridge_not_found, BridgeId}); #{enable := true} -> - emqx_resource:query(ResId, {send_message, Message}); + case emqx_resource:query(ResId, {send_message, Message}) of + {error, {emqx_resource, Reason}} -> throw({bridge_not_ready, Reason}); + Result -> Result + end; #{enable := false} -> throw({bridge_stopped, BridgeId}) end. diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 08c230401..ed1de18cf 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -25,7 +25,7 @@ mod := module(), config := resource_config(), state := resource_state(), - status := started | stopped, + status := started | stopped | starting, metrics := emqx_plugin_libs_metrics:metrics() }. -type resource_group() :: binary(). @@ -41,3 +41,5 @@ %% the `after_query_fun()` is mainly for callbacks that increment counters or do some fallback %% actions upon query failure -type after_query_fun() :: {fun((...) -> ok), Args :: [term()]}. + +-define(TEST_ID_PREFIX, "_test_:"). diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 37c4caa2e..5ec5fd92a 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -82,7 +82,6 @@ ]). -define(HOCON_CHECK_OPTS, #{atom_key => true, nullable => true}). - -define(DEFAULT_RESOURCE_GROUP, <<"default">>). -optional_callbacks([ on_query/4 @@ -170,7 +169,7 @@ create_dry_run(ResourceType, Config) -> -spec create_dry_run_local(resource_type(), resource_config()) -> ok | {error, Reason :: term()}. create_dry_run_local(ResourceType, Config) -> - InstId = iolist_to_binary(emqx_misc:gen_id(16)), + InstId = emqx_resource_instance:make_test_id(), call_instance(InstId, {create_dry_run, InstId, ResourceType, Config}). -spec recreate(instance_id(), resource_type(), resource_config(), term()) -> @@ -201,14 +200,18 @@ query(InstId, Request) -> -spec query(instance_id(), Request :: term(), after_query()) -> Result :: term(). query(InstId, Request, AfterQuery) -> case get_instance(InstId) of + {ok, #{status := starting}} -> + query_error(starting, <<"cannot serve query when the resource " + "instance is still starting">>); {ok, #{status := stopped}} -> - error({resource_stopped, InstId}); + query_error(stopped, <<"cannot serve query when the resource " + "instance is stopped">>); {ok, #{mod := Mod, state := ResourceState, status := started}} -> %% the resource state is readonly to Module:on_query/4 %% and the `after_query()` functions should be thread safe Mod:on_query(InstId, Request, AfterQuery, ResourceState); - {error, Reason} -> - error({get_instance, {InstId, Reason}}) + {error, not_found} -> + query_error(not_found, <<"the resource id not exists">>) end. -spec restart(instance_id()) -> ok | {error, Reason :: term()}. @@ -368,3 +371,6 @@ cluster_call(Func, Args) -> {ok, _TxnId, Result} -> Result; Failed -> Failed end. + +query_error(Reason, Msg) -> + {error, {?MODULE, #{reason => Reason, msg => Msg}}}. diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index 745b8b684..c413691d9 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -26,6 +26,7 @@ -export([ lookup/1 , get_metrics/1 , list_all/0 + , make_test_id/0 ]). -export([ hash_call/2 @@ -61,7 +62,7 @@ hash_call(InstId, Request) -> hash_call(InstId, Request, Timeout) -> gen_server:call(pick(InstId), Request, Timeout). --spec lookup(instance_id()) -> {ok, resource_data()} | {error, Reason :: term()}. +-spec lookup(instance_id()) -> {ok, resource_data()} | {error, not_found}. lookup(InstId) -> case ets:lookup(emqx_resource_instance, InstId) of [] -> {error, not_found}; @@ -69,6 +70,10 @@ lookup(InstId) -> {ok, Data#{id => InstId, metrics => get_metrics(InstId)}} end. +make_test_id() -> + RandId = iolist_to_binary(emqx_misc:gen_id(16)), + <>. + get_metrics(InstId) -> emqx_plugin_libs_metrics:get_metrics(resource_metrics, InstId). @@ -146,7 +151,7 @@ do_recreate(InstId, ResourceType, NewConfig, Params) -> {ok, #{mod := ResourceType, state := ResourceState, config := OldConfig}} -> Config = emqx_resource:call_config_merge(ResourceType, OldConfig, NewConfig, Params), - TestInstId = iolist_to_binary(emqx_misc:gen_id(16)), + TestInstId = make_test_id(), case do_create_dry_run(TestInstId, ResourceType, Config) of ok -> do_remove(ResourceType, InstId, ResourceState, false), @@ -166,7 +171,9 @@ do_create(InstId, ResourceType, Config, Opts) -> {ok, _} -> {ok, already_created}; _ -> Res0 = #{id => InstId, mod => ResourceType, config => Config, - status => stopped, state => undefined}, + status => starting, state => undefined}, + %% The `emqx_resource:call_start/3` need the instance exist beforehand + ets:insert(emqx_resource_instance, {InstId, Res0}), case emqx_resource:call_start(InstId, ResourceType, Config) of {ok, ResourceState} -> ok = emqx_plugin_libs_metrics:create_metrics(resource_metrics, InstId), @@ -181,6 +188,7 @@ do_create(InstId, ResourceType, Config, Opts) -> ets:insert(emqx_resource_instance, {InstId, Res0}), {ok, Res0}; {error, Reason} when ForceCreate == false -> + ets:delete(emqx_resource_instance, InstId), {error, Reason} end end. diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 6b2e5903e..0b65b5a78 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -142,7 +142,7 @@ t_stop_start(_) -> ?assertNot(is_process_alive(Pid0)), - ?assertException(error, {resource_stopped, ?ID}, emqx_resource:query(?ID, get_state)), + ?assertMatch({emqx_resource, #{reason := stopped}}, emqx_resource:query(?ID, get_state)), ok = emqx_resource:restart(?ID), diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 428a1ef8c..049829c59 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -240,7 +240,8 @@ handle_output(RuleId, OutId, Selected, Envs) -> catch Err:Reason:ST -> ok = emqx_plugin_libs_metrics:inc_failed(rule_metrics, RuleId), - ?SLOG(error, #{msg => "output_failed", + Level = case Err of throw -> debug; _ -> error end, + ?SLOG(Level, #{msg => "output_failed", output => OutId, exception => Err, reason => Reason, From f65eca4c4705e56e611ef7b8cbc189a41e4b24eb Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 31 Dec 2021 21:08:07 +0800 Subject: [PATCH 77/99] fix(authn): update testcase for resource not running --- apps/emqx_authn/test/emqx_authn_http_SUITE.erl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl index 2c0716e8b..146e9ef32 100644 --- a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl @@ -153,9 +153,8 @@ t_destroy(_Config) -> ?GLOBAL), % Authenticator should not be usable anymore - ?assertException( - error, - _, + ?assertMatch( + ignore, emqx_authn_http:authenticate( Credentials, State)). From 658f819aabff26a08effb2b5607be36608efad5b Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 31 Dec 2021 21:28:32 +0800 Subject: [PATCH 78/99] fix(bridges): keep multiple bridges from affecting each other on crash --- apps/emqx_bridge/src/emqx_bridge.erl | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 3ae4e5820..3518190c7 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -80,26 +80,35 @@ unload_hook() -> on_message_publish(Message = #message{topic = Topic, flags = Flags}) -> case maps:get(sys, Flags, false) of false -> - lists:foreach(fun (Id) -> - send_message(Id, emqx_rule_events:eventmsg_publish(Message)) - end, get_matched_bridges(Topic)); + Msg = emqx_rule_events:eventmsg_publish(Message), + send_to_egress_matched_bridges(Topic, Msg); true -> ok end, {ok, Message}. +send_to_egress_matched_bridges(Topic, Msg) -> + lists:foreach(fun (Id) -> + try send_message(Id, Msg) of + ok -> ok; + Error -> ?SLOG(error, #{msg => "send_message_to_bridge_failed", + bridge => Id, error => Error}) + catch Err:Reason:ST -> + ?SLOG(error, #{msg => "send_message_to_bridge_crash", + bridge => Id, error => Err, reason => Reason, + stacktrace => ST}) + end + end, get_matched_bridges(Topic)). + send_message(BridgeId, Message) -> {BridgeType, BridgeName} = parse_bridge_id(BridgeId), ResId = emqx_bridge:resource_id(BridgeType, BridgeName), case emqx:get_config([bridges, BridgeType, BridgeName], not_found) of not_found -> - throw({bridge_not_found, BridgeId}); + {error, {bridge_not_found, BridgeId}}; #{enable := true} -> - case emqx_resource:query(ResId, {send_message, Message}) of - {error, {emqx_resource, Reason}} -> throw({bridge_not_ready, Reason}); - Result -> Result - end; + emqx_resource:query(ResId, {send_message, Message}); #{enable := false} -> - throw({bridge_stopped, BridgeId}) + {error, {bridge_stopped, BridgeId}} end. config_key_path() -> From b74a9bfda1d72871fa73a0f3ca94f175e8d665d7 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 31 Dec 2021 22:20:22 +0800 Subject: [PATCH 79/99] fix(swagger): duplicate keys in swagger doc --- apps/emqx_bridge/src/emqx_bridge_http_schema.erl | 4 ---- apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl | 8 -------- apps/emqx_connector/src/emqx_connector_mqtt.erl | 4 ---- 3 files changed, 16 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge/src/emqx_bridge_http_schema.erl index 494911d21..152289cf1 100644 --- a/apps/emqx_bridge/src/emqx_bridge_http_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_http_schema.erl @@ -68,7 +68,6 @@ How long will the HTTP request timeout. fields("post") -> [ type_field() - , name_field() ] ++ fields("bridge"); fields("put") -> @@ -103,8 +102,5 @@ id_field() -> type_field() -> {type, mk(http, #{desc => "The Bridge Type"})}. -name_field() -> - {name, mk(binary(), #{desc => "The Bridge Name"})}. - method() -> enum([post, put, get, delete]). diff --git a/apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl index 3de011b4c..96c9a1d38 100644 --- a/apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl @@ -24,11 +24,9 @@ fields("egress") -> fields("post_ingress") -> [ type_field() - , name_field() ] ++ proplists:delete(enable, fields("ingress")); fields("post_egress") -> [ type_field() - , name_field() ] ++ proplists:delete(enable, fields("egress")); fields("put_ingress") -> @@ -49,9 +47,3 @@ id_field() -> type_field() -> {type, mk(mqtt, #{desc => "The Bridge Type"})}. - -name_field() -> - {name, mk(binary(), - #{ desc => "The Bridge Name" - , example => "some_bridge_name" - })}. diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index c216e905c..beeff6d3e 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -68,10 +68,6 @@ fields("put") -> fields("post") -> [ {type, mk(mqtt, #{desc => "The Connector Type"})} - , {name, mk(binary(), - #{ desc => "The Connector Name" - , example => <<"my_mqtt_connector">> - })} ] ++ fields("put"). %% =================================================================== From efec4564f0a1bbdcdd6d724cbb257069f47e2b9c Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 31 Dec 2021 22:25:45 +0800 Subject: [PATCH 80/99] fix(resource): update test cases on resource not_found --- apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl | 5 ++--- apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl | 5 ++--- apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl | 5 ++--- apps/emqx_authn/test/emqx_authn_redis_SUITE.erl | 5 ++--- apps/emqx_resource/test/emqx_resource_SUITE.erl | 7 +++---- 5 files changed, 11 insertions(+), 16 deletions(-) diff --git a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl index edd91be55..855a2226d 100644 --- a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl @@ -146,9 +146,8 @@ t_destroy(_Config) -> ?GLOBAL), % Authenticator should not be usable anymore - ?assertException( - error, - _, + ?assertMatch( + ignore, emqx_authn_mongodb:authenticate( #{username => <<"plain">>, password => <<"plain">> diff --git a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl index 95eecdead..ffe98be65 100644 --- a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl @@ -159,9 +159,8 @@ t_destroy(_Config) -> ?GLOBAL), % Authenticator should not be usable anymore - ?assertException( - error, - _, + ?assertMatch( + ignore, emqx_authn_mysql:authenticate( #{username => <<"plain">>, password => <<"plain">> diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl index e33f5c100..56044faf4 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl @@ -159,9 +159,8 @@ t_destroy(_Config) -> ?GLOBAL), % Authenticator should not be usable anymore - ?assertException( - error, - _, + ?assertMatch( + ignore, emqx_authn_pgsql:authenticate( #{username => <<"plain">>, password => <<"plain">> diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index de556a7bd..eeb674f86 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -164,9 +164,8 @@ t_destroy(_Config) -> ?GLOBAL), % Authenticator should not be usable anymore - ?assertException( - error, - _, + ?assertMatch( + ignore, emqx_authn_redis:authenticate( #{username => <<"plain">>, password => <<"plain">> diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 0b65b5a78..4e9c35efc 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -96,9 +96,7 @@ t_query(_) -> ?assert(false) end, - ?assertException( - error, - {get_instance, _Reason}, + ?assertMatch({error, {emqx_resource, #{reason := not_found}}}, emqx_resource:query(<<"unknown">>, get_state)), ok = emqx_resource:remove_local(?ID). @@ -142,7 +140,8 @@ t_stop_start(_) -> ?assertNot(is_process_alive(Pid0)), - ?assertMatch({emqx_resource, #{reason := stopped}}, emqx_resource:query(?ID, get_state)), + ?assertMatch({error, {emqx_resource, #{reason := stopped}}}, + emqx_resource:query(?ID, get_state)), ok = emqx_resource:restart(?ID), From 9a7452e1c5abe778a89711d582a1bdd8162d3f0a Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 1 Jan 2022 03:07:31 +0800 Subject: [PATCH 81/99] fix(connector): add testcase for binding ingress mqtt bridge to rules --- apps/emqx_bridge/src/emqx_bridge.erl | 4 +- .../src/mqtt/emqx_connector_mqtt_mod.erl | 63 ++++++++++---- .../src/mqtt/emqx_connector_mqtt_msg.erl | 30 +------ .../test/emqx_connector_api_SUITE.erl | 82 +++++++++++++++++-- 4 files changed, 125 insertions(+), 54 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 3518190c7..0495f00e4 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -81,12 +81,12 @@ on_message_publish(Message = #message{topic = Topic, flags = Flags}) -> case maps:get(sys, Flags, false) of false -> Msg = emqx_rule_events:eventmsg_publish(Message), - send_to_egress_matched_bridges(Topic, Msg); + send_to_matched_egress_bridges(Topic, Msg); true -> ok end, {ok, Message}. -send_to_egress_matched_bridges(Topic, Msg) -> +send_to_matched_egress_bridges(Topic, Msg) -> lists:foreach(fun (Id) -> try send_message(Id, Msg) of ok -> ok; diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl index 30a1ccb30..7d5021f82 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl @@ -165,7 +165,8 @@ handle_publish(Msg, undefined) -> ?SLOG(error, #{msg => "cannot_publish_to_local_broker_as" "_'ingress'_is_not_configured", message => Msg}); -handle_publish(Msg, Vars) -> +handle_publish(Msg0, Vars) -> + Msg = format_msg_received(Msg0), ?SLOG(debug, #{msg => "publish_to_local_broker", message => Msg, vars => Vars}), case Vars of @@ -173,27 +174,11 @@ handle_publish(Msg, Vars) -> _ = erlang:apply(Mod, Func, [Msg | Args]); _ -> ok end, - maybe_publish_to_local_broker(Msg, Vars). + maybe_publish_to_local_broker(Msg0, Vars). handle_disconnected(Reason, Parent) -> Parent ! {disconnected, self(), Reason}. -maybe_publish_to_local_broker(#{topic := Topic} = Msg, #{remote_topic := SubTopic} = Vars) -> - case maps:get(local_topic, Vars, undefined) of - undefined -> - %% local topic is not set, discard it - ok; - _ -> - case emqx_topic:match(Topic, SubTopic) of - true -> - _ = emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars)), - ok; - false -> - ?SLOG(warning, #{msg => "discard_message_as_topic_not_matched", - message => Msg, subscribed => SubTopic, got_topic => Topic}) - end - end. - make_hdlr(Parent, Vars) -> #{puback => {fun ?MODULE:handle_puback/2, [Parent]}, publish => {fun ?MODULE:handle_publish/2, [Vars]}, @@ -209,3 +194,45 @@ sub_remote_topics(ClientPid, #{remote_topic := FromTopic, remote_qos := QoS}) -> process_config(Config) -> maps:without([conn_type, address, receive_mountpoint, subscriptions, name], Config). + +maybe_publish_to_local_broker(#{topic := Topic} = Msg, #{remote_topic := SubTopic} = Vars) -> + case maps:get(local_topic, Vars, undefined) of + undefined -> + ok; %% local topic is not set, discard it + _ -> + case emqx_topic:match(Topic, SubTopic) of + true -> + _ = emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars)), + ok; + false -> + ?SLOG(warning, #{msg => "discard_message_as_topic_not_matched", + message => Msg, subscribed => SubTopic, got_topic => Topic}) + end + end. + +format_msg_received(#{dup := Dup, payload := Payload, properties := Props, + qos := QoS, retain := Retain, topic := Topic}) -> + #{event => '$bridges/mqtt', + id => emqx_guid:to_hexstr(emqx_guid:gen()), + payload => Payload, + topic => Topic, + qos => QoS, + dup => Dup, + retain => Retain, + pub_props => printable_maps(Props), + timestamp => erlang:system_time(millisecond) + }. + +printable_maps(undefined) -> #{}; +printable_maps(Headers) -> + maps:fold( + fun ('User-Property', V0, AccIn) when is_list(V0) -> + AccIn#{ + 'User-Property' => maps:from_list(V0), + 'User-Property-Pairs' => [#{ + key => Key, + value => Value + } || {Key, Value} <- V0] + }; + (K, V0, AccIn) -> AccIn#{K => V0} + end, #{}, Headers). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl index a0dd9eec1..35bcf3de1 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -78,10 +78,9 @@ to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) -> Msg#message{topic = topic(Mountpoint, Topic)}. %% published from remote node over a MQTT connection -to_broker_msg(#{dup := Dup, properties := Props} = MapMsg0, +to_broker_msg(#{dup := Dup, properties := Props} = MapMsg, #{local_topic := TopicToken, payload := PayloadToken, local_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) -> - MapMsg = format_msg_received(MapMsg0), Topic = replace_vars_in_str(TopicToken, MapMsg), Payload = process_payload(PayloadToken, MapMsg), QoS = replace_simple_var(QoSToken, MapMsg), @@ -90,33 +89,6 @@ to_broker_msg(#{dup := Dup, properties := Props} = MapMsg0, emqx_message:set_flags(#{dup => Dup, retain => Retain}, emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload))). -format_msg_received(#{dup := Dup, payload := Payload, properties := Props, - qos := QoS, retain := Retain, topic := Topic}) -> - #{event => '$bridges/mqtt', - id => emqx_guid:to_hexstr(emqx_guid:gen()), - payload => Payload, - topic => Topic, - qos => QoS, - flags => #{dup => Dup, retain => Retain}, - pub_props => printable_maps(Props), - timestamp => erlang:system_time(millisecond), - node => node() - }. - -printable_maps(undefined) -> #{}; -printable_maps(Headers) -> - maps:fold( - fun ('User-Property', V0, AccIn) when is_list(V0) -> - AccIn#{ - 'User-Property' => maps:from_list(V0), - 'User-Property-Pairs' => [#{ - key => Key, - value => Value - } || {Key, Value} <- V0] - }; - (K, V0, AccIn) -> AccIn#{K => V0} - end, #{}, Headers). - process_payload([], Msg) -> emqx_json:encode(Msg); process_payload(Tks, Msg) -> diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index 1a96a3596..96d793640 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -22,7 +22,10 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --define(CONF_DEFAULT, <<"connectors: {}">>). +%% output functions +-export([ inspect/3 + ]). + -define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>). -define(CONNECTR_TYPE, <<"mqtt">>). -define(CONNECTR_NAME, <<"test_connector">>). @@ -67,6 +70,9 @@ <<"failed">> := FAILED, <<"rate">> := SPEED, <<"rate_last5m">> := SPEED5M, <<"rate_max">> := SPEEDMAX}). +inspect(Selected, _Envs, _Args) -> + persistent_term:put(?MODULE, #{inspect => Selected}). + all() -> emqx_common_test_helpers:all(?MODULE). @@ -89,13 +95,15 @@ init_per_suite(Config) -> %% some testcases (may from other app) already get emqx_connector started _ = application:stop(emqx_resource), _ = application:stop(emqx_connector), - ok = emqx_common_test_helpers:start_apps([emqx_connector, emqx_bridge, emqx_dashboard]), - ok = emqx_config:init_load(emqx_connector_schema, ?CONF_DEFAULT), + ok = emqx_common_test_helpers:start_apps([emqx_rule_engine, emqx_connector, + emqx_bridge, emqx_dashboard]), + ok = emqx_config:init_load(emqx_connector_schema, <<"connectors: {}">>), + ok = emqx_config:init_load(emqx_rule_engine_schema, <<"rule_engine {rules {}}">>), ok = emqx_config:init_load(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT), Config. end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([emqx_connector, emqx_bridge, emqx_dashboard]), + emqx_common_test_helpers:stop_apps([emqx_rule_engine, emqx_connector, emqx_bridge, emqx_dashboard]), ok. init_per_testcase(_, Config) -> @@ -223,7 +231,6 @@ t_mqtt_conn_bridge_ingress(_) -> %% PUBLISH a message to the 'remote' broker, as we have only one broker, %% the remote broker is also the local one. emqx:publish(emqx_message:make(RemoteTopic, Payload)), - %% we should receive a message on the local broker, with specified topic ?assert( receive @@ -435,6 +442,71 @@ t_mqtt_conn_testing(_) -> <<"name">> => ?BRIDGE_NAME_EGRESS }). +t_ingress_mqtt_bridge_with_rules(_) -> + {ok, 201, Connector} = request(post, uri(["connectors"]), + ?MQTT_CONNECOTR(<<"user1">>)#{ <<"type">> => ?CONNECTR_TYPE + , <<"name">> => ?CONNECTR_NAME + }), + #{ <<"id">> := ConnctorID } = jsx:decode(Connector), + + {ok, 201, Bridge} = request(post, uri(["bridges"]), + ?MQTT_BRIDGE_INGRESS(ConnctorID)#{ + <<"type">> => ?CONNECTR_TYPE, + <<"name">> => ?BRIDGE_NAME_INGRESS + }), + #{ <<"id">> := BridgeIDIngress } = jsx:decode(Bridge), + + {ok, 201, Rule} = request(post, uri(["rules"]), + #{<<"name">> => <<"A rule get messages from a source mqtt bridge">>, + <<"enable">> => true, + <<"outputs">> => [#{<<"function">> => "emqx_connector_api_SUITE:inspect"}], + <<"sql">> => <<"SELECT * from \"$bridges/", BridgeIDIngress/binary, "\"">> + }), + #{<<"id">> := RuleId} = jsx:decode(Rule), + + %% we now test if the bridge works as expected + + RemoteTopic = <<"remote_topic/1">>, + LocalTopic = <<"local_topic/", RemoteTopic/binary>>, + Payload = <<"hello">>, + emqx:subscribe(LocalTopic), + %% PUBLISH a message to the 'remote' broker, as we have only one broker, + %% the remote broker is also the local one. + emqx:publish(emqx_message:make(RemoteTopic, Payload)), + %% we should receive a message on the local broker, with specified topic + ?assert( + receive + {deliver, LocalTopic, #message{payload = Payload}} -> + ct:pal("local broker got message: ~p on topic ~p", [Payload, LocalTopic]), + true; + Msg -> + ct:pal("Msg: ~p", [Msg]), + false + after 100 -> + false + end), + %% and also the rule should be matched, with matched + 1: + {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []), + #{ <<"id">> := RuleId + , <<"metrics">> := #{<<"matched">> := 1} + } = jsx:decode(Rule1), + %% we also check if the outputs of the rule is triggered + ?assertMatch(#{inspect := #{ + event := '$bridges/mqtt', + id := MsgId, + payload := Payload, + topic := RemoteTopic, + qos := 0, + dup := false, + retain := false, + pub_props := #{}, + timestamp := _ + }} when is_binary(MsgId), persistent_term:get(?MODULE)), + + {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []), + {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []). + %%-------------------------------------------------------------------- %% HTTP Request %%-------------------------------------------------------------------- From 925d46fe86433aef4973424e8403239ddd00673e Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 1 Jan 2022 04:12:20 +0800 Subject: [PATCH 82/99] fix(connector): add testcase for binding egress mqtt bridge to rules --- apps/emqx_bridge/src/emqx_bridge.erl | 15 +++ .../test/emqx_connector_api_SUITE.erl | 91 +++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 0495f00e4..2e610b2b9 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -39,11 +39,14 @@ , lookup/3 , list/0 , list_bridges_by_connector/1 + , create/2 , create/3 , recreate/2 , recreate/3 , create_dry_run/2 + , remove/1 , remove/3 + , update/2 , update/3 , start/2 , stop/2 @@ -207,6 +210,10 @@ stop(Type, Name) -> restart(Type, Name) -> emqx_resource:restart(resource_id(Type, Name)). +create(BridgeId, Conf) -> + {BridgeType, BridgeName} = parse_bridge_id(BridgeId), + create(BridgeType, BridgeName, Conf). + create(Type, Name, Conf) -> ?SLOG(info, #{msg => "create bridge", type => Type, name => Name, config => Conf}), @@ -217,6 +224,10 @@ create(Type, Name, Conf) -> {error, Reason} -> {error, Reason} end. +update(BridgeId, {OldConf, Conf}) -> + {BridgeType, BridgeName} = parse_bridge_id(BridgeId), + update(BridgeType, BridgeName, {OldConf, Conf}). + update(Type, Name, {OldConf, Conf}) -> %% TODO: sometimes its not necessary to restart the bridge connection. %% @@ -263,6 +274,10 @@ create_dry_run(Type, Conf) -> Error end. +remove(BridgeId) -> + {BridgeType, BridgeName} = parse_bridge_id(BridgeId), + remove(BridgeType, BridgeName, #{}). + remove(Type, Name, _Conf) -> ?SLOG(info, #{msg => "remove_bridge", type => Type, name => Name}), case emqx_resource:remove_local(resource_id(Type, Name)) of diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index 96d793640..936982e75 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -110,8 +110,20 @@ init_per_testcase(_, Config) -> {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), Config. end_per_testcase(_, _Config) -> + clear_resources(), ok. +clear_resources() -> + lists:foreach(fun(#{id := Id}) -> + ok = emqx_rule_engine:delete_rule(Id) + end, emqx_rule_engine:get_rules()), + lists:foreach(fun(#{id := Id}) -> + ok = emqx_bridge:remove(Id) + end, emqx_bridge:list()), + lists:foreach(fun(#{<<"id">> := Id}) -> + ok = emqx_connector:delete(Id) + end, emqx_connector:list()). + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -507,6 +519,85 @@ t_ingress_mqtt_bridge_with_rules(_) -> {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []), {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []). +t_egress_mqtt_bridge_with_rules(_) -> + {ok, 201, Connector} = request(post, uri(["connectors"]), + ?MQTT_CONNECOTR(<<"user1">>)#{ <<"type">> => ?CONNECTR_TYPE + , <<"name">> => ?CONNECTR_NAME + }), + #{ <<"id">> := ConnctorID } = jsx:decode(Connector), + + {ok, 201, Bridge} = request(post, uri(["bridges"]), + ?MQTT_BRIDGE_EGRESS(ConnctorID)#{ + <<"type">> => ?CONNECTR_TYPE, + <<"name">> => ?BRIDGE_NAME_EGRESS + }), + #{ <<"id">> := BridgeIDEgress } = jsx:decode(Bridge), + + {ok, 201, Rule} = request(post, uri(["rules"]), + #{<<"name">> => <<"A rule send messages to a sink mqtt bridge">>, + <<"enable">> => true, + <<"outputs">> => [BridgeIDEgress], + <<"sql">> => <<"SELECT * from \"t/1\"">> + }), + #{<<"id">> := RuleId} = jsx:decode(Rule), + + %% we now test if the bridge works as expected + LocalTopic = <<"local_topic/1">>, + RemoteTopic = <<"remote_topic/", LocalTopic/binary>>, + Payload = <<"hello">>, + emqx:subscribe(RemoteTopic), + %% PUBLISH a message to the 'local' broker, as we have only one broker, + %% the remote broker is also the local one. + emqx:publish(emqx_message:make(LocalTopic, Payload)), + %% we should receive a message on the "remote" broker, with specified topic + ?assert( + receive + {deliver, RemoteTopic, #message{payload = Payload}} -> + ct:pal("local broker got message: ~p on topic ~p", [Payload, RemoteTopic]), + true; + Msg -> + ct:pal("Msg: ~p", [Msg]), + false + after 100 -> + false + end), + emqx:unsubscribe(RemoteTopic), + + %% PUBLISH a message to the rule. + Payload2 = <<"hi">>, + RuleTopic = <<"t/1">>, + RemoteTopic2 = <<"remote_topic/", RuleTopic/binary>>, + emqx:subscribe(RemoteTopic2), + emqx:publish(emqx_message:make(RuleTopic, Payload2)), + {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []), + #{ <<"id">> := RuleId + , <<"metrics">> := #{<<"matched">> := 1} + } = jsx:decode(Rule1), + %% we should receive a message on the "remote" broker, with specified topic + ?assert( + receive + {deliver, RemoteTopic2, #message{payload = Payload2}} -> + ct:pal("local broker got message: ~p on topic ~p", [Payload2, RemoteTopic2]), + true; + Msg -> + ct:pal("Msg: ~p", [Msg]), + false + after 100 -> + false + end), + + %% verify the metrics of the bridge + {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []), + ?assertMatch(#{ <<"id">> := BridgeIDEgress + , <<"metrics">> := ?metrics(2, 2, 0, _, _, _) + , <<"node_metrics">> := + [#{<<"node">> := _, <<"metrics">> := ?metrics(2, 2, 0, _, _, _)}] + }, jsx:decode(BridgeStr)), + + {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []), + {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []). + %%-------------------------------------------------------------------- %% HTTP Request %%-------------------------------------------------------------------- From 59e2614574b6cff5837a6524b4c07f98930d8edb Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 1 Jan 2022 04:23:51 +0800 Subject: [PATCH 83/99] fix(dialyzer): unmatched results in emqx_statsd_api --- apps/emqx_statsd/src/emqx_statsd_api.erl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl index 2278fa492..d545003b0 100644 --- a/apps/emqx_statsd/src/emqx_statsd_api.erl +++ b/apps/emqx_statsd/src/emqx_statsd_api.erl @@ -59,12 +59,10 @@ statsd(put, #{body := Body}) -> Body, #{rawconf_with_defaults => true, override_to => cluster}) of {ok, #{raw_config := NewConfig, config := Config}} -> + _ = emqx_statsd_sup:stop_child(?APP), case maps:get(<<"enable">>, Body) of - true -> - _ = emqx_statsd_sup:stop_child(?APP), - emqx_statsd_sup:start_child(?APP, maps:get(config, Config)); - false -> - _ = emqx_statsd_sup:stop_child(?APP) + true -> emqx_statsd_sup:start_child(?APP, maps:get(config, Config)); + false -> ok end, {200, NewConfig}; {error, Reason} -> From 808646c2a173a3982bd7c73cd07c7b155b09c8dc Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 1 Jan 2022 15:47:27 +0800 Subject: [PATCH 84/99] fix(bridge): prohibit deleting connectors that are in use --- apps/emqx_bridge/src/emqx_bridge.erl | 2 +- apps/emqx_connector/src/emqx_connector.erl | 38 ++++++++------- .../emqx_connector/src/emqx_connector_api.erl | 4 ++ .../test/emqx_connector_api_SUITE.erl | 47 ++++++++++++------- 4 files changed, 56 insertions(+), 35 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 2e610b2b9..6e014f2ec 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -259,7 +259,7 @@ update(Type, Name, {OldConf, Conf}) -> end. recreate(Type, Name) -> - recreate(Type, Name, emqx:get_raw_config([bridges, Type, Name])). + recreate(Type, Name, emqx:get_config([bridges, Type, Name])). recreate(Type, Name, Conf) -> emqx_resource:recreate_local(resource_id(Type, Name), diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl index 940e958e3..db1caefbb 100644 --- a/apps/emqx_connector/src/emqx_connector.erl +++ b/apps/emqx_connector/src/emqx_connector.erl @@ -37,31 +37,26 @@ config_key_path() -> [connectors]. +-dialyzer([{nowarn_function, [post_config_update/5]}, error_handling]). post_config_update([connectors, Type, Name], '$remove', _, _OldConf, _AppEnvs) -> ConnId = connector_id(Type, Name), - LinkedBridgeIds = lists:foldl(fun - (#{id := BId, raw_config := #{<<"connector">> := ConnId0}}, Acc) - when ConnId0 == ConnId -> - [BId | Acc]; - (_, Acc) -> Acc - end, [], emqx_bridge:list()), - case LinkedBridgeIds of - [] -> ok; - _ -> {error, {dependency_bridges_exist, LinkedBridgeIds}} + try foreach_linked_bridges(ConnId, fun(#{id := BId}) -> + throw({dependency_bridges_exist, BId}) + end) + catch throw:Error -> {error, Error} end; -post_config_update([connectors, Type, Name], _Req, NewConf, _OldConf, _AppEnvs) -> +post_config_update([connectors, Type, Name], _Req, NewConf, OldConf, _AppEnvs) -> ConnId = connector_id(Type, Name), - lists:foreach(fun - (#{id := BId, raw_config := #{<<"connector">> := ConnId0}}) when ConnId0 == ConnId -> + foreach_linked_bridges(ConnId, + fun(#{id := BId}) -> {BType, BName} = emqx_bridge:parse_bridge_id(BId), BridgeConf = emqx:get_config([bridges, BType, BName]), - case emqx_bridge:recreate(BType, BName, BridgeConf#{connector => NewConf}) of - {ok, _} -> ok; + case emqx_bridge:update(BType, BName, {BridgeConf#{connector => OldConf}, + BridgeConf#{connector => NewConf}}) of + ok -> ok; {error, Reason} -> error({update_bridge_error, Reason}) - end; - (_) -> - ok - end, emqx_bridge:list()). + end + end). connector_id(Type0, Name0) -> Type = bin(Type0), @@ -112,3 +107,10 @@ delete(Type, Name) -> bin(Bin) when is_binary(Bin) -> Bin; bin(Str) when is_list(Str) -> list_to_binary(Str); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). + +foreach_linked_bridges(ConnId, Do) -> + lists:foreach(fun + (#{raw_config := #{<<"connector">> := ConnId0}} = Bridge) when ConnId0 == ConnId -> + Do(Bridge); + (_) -> ok + end, emqx_bridge:list()). diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index 4989cf17e..72938649c 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -253,6 +253,10 @@ schema("/connectors/:id") -> {ok, _} -> case emqx_connector:delete(ConnType, ConnName) of {ok, _} -> {204}; + {error, {post_config_update, _, {dependency_bridges_exist, BridgeID}}} -> + {403, error_msg('DEPENDENCY_EXISTS', + <<"Cannot remove the connector as it's in use by a bridge: ", + BridgeID/binary>>)}; {error, Error} -> {400, error_msg('BAD_ARG', Error)} end; {error, not_found} -> diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index 936982e75..4caf700a9 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -108,6 +108,9 @@ end_per_suite(_Config) -> init_per_testcase(_, Config) -> {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), + %% assert we there's no connectors and no bridges at first + {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []), + {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), Config. end_per_testcase(_, _Config) -> clear_resources(), @@ -200,10 +203,6 @@ t_mqtt_crud_apis(_) -> ok. t_mqtt_conn_bridge_ingress(_) -> - %% assert we there's no connectors and no bridges at first - {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []), - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), - %% then we add a mqtt connector, using POST User1 = <<"user1">>, {ok, 201, Connector} = request(post, uri(["connectors"]), @@ -272,10 +271,6 @@ t_mqtt_conn_bridge_ingress(_) -> ok. t_mqtt_conn_bridge_egress(_) -> - %% assert we there's no connectors and no bridges at first - {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []), - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), - %% then we add a mqtt connector, using POST User1 = <<"user1">>, {ok, 201, Connector} = request(post, uri(["connectors"]), @@ -350,10 +345,6 @@ t_mqtt_conn_bridge_egress(_) -> %% - update a connector should also update all of the the bridges %% - cannot delete a connector that is used by at least one bridge t_mqtt_conn_update(_) -> - %% assert we there's no connectors and no bridges at first - {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []), - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), - %% then we add a mqtt connector, using POST {ok, 201, Connector} = request(post, uri(["connectors"]), ?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>) @@ -396,10 +387,6 @@ t_mqtt_conn_update(_) -> {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []). t_mqtt_conn_update2(_) -> - %% assert we there's no connectors and no bridges at first - {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []), - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), - %% then we add a mqtt connector, using POST %% but this connector is point to a unreachable server "2603" {ok, 201, Connector} = request(post, uri(["connectors"]), @@ -440,6 +427,34 @@ t_mqtt_conn_update2(_) -> {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []), {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []). +t_mqtt_conn_update3(_) -> + %% we add a mqtt connector, using POST + {ok, 201, Connector} = request(post, uri(["connectors"]), + ?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>) + #{ <<"type">> => ?CONNECTR_TYPE + , <<"name">> => ?CONNECTR_NAME + }), + #{ <<"id">> := ConnctorID } = jsx:decode(Connector), + + %% ... and a MQTT bridge, using POST + %% we bind this bridge to the connector created just now + {ok, 201, Bridge} = request(post, uri(["bridges"]), + ?MQTT_BRIDGE_EGRESS(ConnctorID)#{ + <<"type">> => ?CONNECTR_TYPE, + <<"name">> => ?BRIDGE_NAME_EGRESS + }), + #{ <<"id">> := BridgeIDEgress + , <<"status">> := <<"connected">> + , <<"connector">> := ConnctorID + } = jsx:decode(Bridge), + + %% delete the connector should fail because it is in use by a bridge + {ok, 403, _} = request(delete, uri(["connectors", ConnctorID]), []), + %% delete the bridge + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []), + %% the connector now can be deleted without problems + {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []). + t_mqtt_conn_testing(_) -> %% APIs for testing the connectivity %% then we add a mqtt connector, using POST From 2277b75b2f01d754c6bcb0402072d988a7854618 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sun, 2 Jan 2022 16:41:50 +0800 Subject: [PATCH 85/99] refactor(resource): improve the process starting/stopping resource instances --- apps/emqx_resource/src/emqx_resource.erl | 46 +++--- .../src/emqx_resource_health_check.erl | 24 +-- .../src/emqx_resource_health_check_sup.erl | 37 +++-- .../src/emqx_resource_instance.erl | 153 ++++++++---------- apps/emqx_resource/src/emqx_resource_sup.erl | 2 +- 5 files changed, 127 insertions(+), 135 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 5ec5fd92a..12ae912e8 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -58,6 +58,7 @@ %% Calls to the callback module with current resource state %% They also save the state after the call finished (except query/2,3). -export([ restart/1 %% restart the instance. + , restart/2 , health_check/1 %% verify if the resource is working normally , stop/1 %% stop the instance , query/2 %% query the instance @@ -68,7 +69,6 @@ -export([ call_start/3 %% start the instance , call_health_check/3 %% verify if the resource is working normally , call_stop/3 %% stop the instance - , call_config_merge/4 %% merge the config when updating , call_jsonify/2 ]). @@ -86,12 +86,9 @@ -optional_callbacks([ on_query/4 , on_health_check/2 - , on_config_merge/3 , on_jsonify/1 ]). --callback on_config_merge(resource_config(), resource_config(), term()) -> resource_config(). - -callback on_jsonify(resource_config()) -> jsx:json_term(). %% when calling emqx_resource:start/1 @@ -169,18 +166,17 @@ create_dry_run(ResourceType, Config) -> -spec create_dry_run_local(resource_type(), resource_config()) -> ok | {error, Reason :: term()}. create_dry_run_local(ResourceType, Config) -> - InstId = emqx_resource_instance:make_test_id(), - call_instance(InstId, {create_dry_run, InstId, ResourceType, Config}). + call_instance(<>, {create_dry_run, ResourceType, Config}). --spec recreate(instance_id(), resource_type(), resource_config(), term()) -> +-spec recreate(instance_id(), resource_type(), resource_config(), create_opts()) -> {ok, resource_data()} | {error, Reason :: term()}. -recreate(InstId, ResourceType, Config, Params) -> - cluster_call(recreate_local, [InstId, ResourceType, Config, Params]). +recreate(InstId, ResourceType, Config, Opts) -> + cluster_call(recreate_local, [InstId, ResourceType, Config, Opts]). --spec recreate_local(instance_id(), resource_type(), resource_config(), term()) -> +-spec recreate_local(instance_id(), resource_type(), resource_config(), create_opts()) -> {ok, resource_data()} | {error, Reason :: term()}. -recreate_local(InstId, ResourceType, Config, Params) -> - call_instance(InstId, {recreate, InstId, ResourceType, Config, Params}). +recreate_local(InstId, ResourceType, Config, Opts) -> + call_instance(InstId, {recreate, InstId, ResourceType, Config, Opts}). -spec remove(instance_id()) -> ok | {error, Reason :: term()}. remove(InstId) -> @@ -216,7 +212,11 @@ query(InstId, Request, AfterQuery) -> -spec restart(instance_id()) -> ok | {error, Reason :: term()}. restart(InstId) -> - call_instance(InstId, {restart, InstId}). + restart(InstId, #{}). + +-spec restart(instance_id(), create_opts()) -> ok | {error, Reason :: term()}. +restart(InstId, Opts) -> + call_instance(InstId, {restart, InstId, Opts}). -spec stop(instance_id()) -> ok | {error, Reason :: term()}. stop(InstId) -> @@ -276,14 +276,6 @@ call_health_check(InstId, Mod, ResourceState) -> call_stop(InstId, Mod, ResourceState) -> ?SAFE_CALL(Mod:on_stop(InstId, ResourceState)). --spec call_config_merge(module(), resource_config(), resource_config(), term()) -> - resource_config(). -call_config_merge(Mod, OldConfig, NewConfig, Params) -> - case erlang:function_exported(Mod, on_config_merge, 3) of - true -> ?SAFE_CALL(Mod:on_config_merge(OldConfig, NewConfig, Params)); - false -> NewConfig - end. - -spec call_jsonify(module(), resource_config()) -> jsx:json_term(). call_jsonify(Mod, Config) -> case erlang:function_exported(Mod, on_jsonify, 1) of @@ -330,17 +322,17 @@ check_and_create_local(InstId, ResourceType, RawConfig, Opts) -> check_and_do(ResourceType, RawConfig, fun(InstConf) -> create_local(InstId, ResourceType, InstConf, Opts) end). --spec check_and_recreate(instance_id(), resource_type(), raw_resource_config(), term()) -> +-spec check_and_recreate(instance_id(), resource_type(), raw_resource_config(), create_opts()) -> {ok, resource_data()} | {error, term()}. -check_and_recreate(InstId, ResourceType, RawConfig, Params) -> +check_and_recreate(InstId, ResourceType, RawConfig, Opts) -> check_and_do(ResourceType, RawConfig, - fun(InstConf) -> recreate(InstId, ResourceType, InstConf, Params) end). + fun(InstConf) -> recreate(InstId, ResourceType, InstConf, Opts) end). --spec check_and_recreate_local(instance_id(), resource_type(), raw_resource_config(), term()) -> +-spec check_and_recreate_local(instance_id(), resource_type(), raw_resource_config(), create_opts()) -> {ok, resource_data()} | {error, term()}. -check_and_recreate_local(InstId, ResourceType, RawConfig, Params) -> +check_and_recreate_local(InstId, ResourceType, RawConfig, Opts) -> check_and_do(ResourceType, RawConfig, - fun(InstConf) -> recreate_local(InstId, ResourceType, InstConf, Params) end). + fun(InstConf) -> recreate_local(InstId, ResourceType, InstConf, Opts) end). check_and_do(ResourceType, RawConfig, Do) when is_function(Do) -> case check_config(ResourceType, RawConfig) of diff --git a/apps/emqx_resource/src/emqx_resource_health_check.erl b/apps/emqx_resource/src/emqx_resource_health_check.erl index 50b236daa..dc0795fb3 100644 --- a/apps/emqx_resource/src/emqx_resource_health_check.erl +++ b/apps/emqx_resource/src/emqx_resource_health_check.erl @@ -15,29 +15,21 @@ %%-------------------------------------------------------------------- -module(emqx_resource_health_check). --export([child_spec/2]). - -export([start_link/2]). -export([health_check/2]). -child_spec(Name, Sleep) -> - #{id => {health_check, Name}, - start => {?MODULE, start_link, [Name, Sleep]}, - restart => transient, - shutdown => 5000, type => worker, modules => [?MODULE]}. - -start_link(Name, Sleep) -> +start_link(Name, Sleep) -> Pid = proc_lib:spawn_link(?MODULE, health_check, [Name, Sleep]), {ok, Pid}. -health_check(Name, SleepTime) -> +health_check(Name, SleepTime) -> timer:sleep(SleepTime), - case emqx_resource:health_check(Name) of - ok -> + case emqx_resource:health_check(Name) of + ok -> emqx_alarm:deactivate(Name); - {error, _} -> - emqx_alarm:activate(Name, #{name => Name}, - <>) + {error, _} -> + emqx_alarm:activate(Name, #{name => Name}, + <>) end, - health_check(Name, SleepTime). \ No newline at end of file + health_check(Name, SleepTime). diff --git a/apps/emqx_resource/src/emqx_resource_health_check_sup.erl b/apps/emqx_resource/src/emqx_resource_health_check_sup.erl index 571cd6338..6a2b07e94 100644 --- a/apps/emqx_resource/src/emqx_resource_health_check_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_health_check_sup.erl @@ -20,8 +20,17 @@ -export([start_link/0]). -export([init/1, - create_health_check_process/2, - delete_health_check_process/1]). + create_checker/2, + delete_checker/1]). + +-define(HEALTH_CHECK_MOD, emqx_resource_health_check). +-define(ID(NAME), {resource_health_check, NAME}). + +child_spec(Name, Sleep) -> + #{id => ?ID(Name), + start => {?HEALTH_CHECK_MOD, start_link, [Name, Sleep]}, + restart => transient, + shutdown => 5000, type => worker, modules => [?HEALTH_CHECK_MOD]}. start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). @@ -30,11 +39,21 @@ init([]) -> SupFlags = #{strategy => one_for_one, intensity => 10, period => 10}, {ok, {SupFlags, []}}. -create_health_check_process(Name, Sleep) -> - supervisor:start_child(emqx_resource_health_check_sup, - emqx_resource_health_check:child_spec(Name, Sleep)). +create_checker(Name, Sleep) -> + case supervisor:start_child(?MODULE, child_spec(Name, Sleep)) of + {ok, _} -> ok; + {error, already_present} -> ok; + {error, {already_started, _}} -> ok; + Error -> Error + end. -delete_health_check_process(Name) -> - _ = supervisor:terminate_child(emqx_resource_health_check_sup, {health_check, Name}), - _ = supervisor:delete_child(emqx_resource_health_check_sup, {health_check, Name}), - ok. \ No newline at end of file +delete_checker(Name) -> + case supervisor:terminate_child(?MODULE, {health_check, Name}) of + ok -> + case supervisor:delete_child(?MODULE, {health_check, Name}) of + {error, not_found} -> ok; + Error -> Error + end; + {error, not_found} -> ok; + Error -> Error + end. diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index b2f2d0d27..2701b70d6 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -103,17 +103,17 @@ init({Pool, Id}) -> handle_call({create, InstId, ResourceType, Config, Opts}, _From, State) -> {reply, do_create(InstId, ResourceType, Config, Opts), State}; -handle_call({create_dry_run, InstId, ResourceType, Config}, _From, State) -> - {reply, do_create_dry_run(InstId, ResourceType, Config), State}; +handle_call({create_dry_run, ResourceType, Config}, _From, State) -> + {reply, do_create_dry_run(ResourceType, Config), State}; -handle_call({recreate, InstId, ResourceType, Config, Params}, _From, State) -> - {reply, do_recreate(InstId, ResourceType, Config, Params), State}; +handle_call({recreate, InstId, ResourceType, Config, Opts}, _From, State) -> + {reply, do_recreate(InstId, ResourceType, Config, Opts), State}; handle_call({remove, InstId}, _From, State) -> {reply, do_remove(InstId), State}; -handle_call({restart, InstId}, _From, State) -> - {reply, do_restart(InstId), State}; +handle_call({restart, InstId, Opts}, _From, State) -> + {reply, do_restart(InstId, Opts), State}; handle_call({stop, InstId}, _From, State) -> {reply, do_stop(InstId), State}; @@ -142,20 +142,17 @@ code_change(_OldVsn, State, _Extra) -> %% suppress the race condition check, as these functions are protected in gproc workers -dialyzer({nowarn_function, [do_recreate/4, do_create/4, - do_restart/1, + do_restart/2, do_stop/1, do_health_check/1]}). -do_recreate(InstId, ResourceType, NewConfig, Params) -> +do_recreate(InstId, ResourceType, NewConfig, Opts) -> case lookup(InstId) of - {ok, #{mod := ResourceType, state := ResourceState, config := OldConfig}} -> - Config = emqx_resource:call_config_merge(ResourceType, OldConfig, - NewConfig, Params), - TestInstId = make_test_id(), - case do_create_dry_run(TestInstId, ResourceType, Config) of + {ok, #{mod := ResourceType} = Data} -> + case do_create_dry_run(ResourceType, NewConfig) of ok -> - do_remove(ResourceType, InstId, ResourceState, false), - do_create(InstId, ResourceType, Config, #{force_create => true}); + do_remove(Data, false), + do_create(InstId, ResourceType, NewConfig, Opts#{force_create => true}); Error -> Error end; @@ -166,100 +163,86 @@ do_recreate(InstId, ResourceType, NewConfig, Params) -> end. do_create(InstId, ResourceType, Config, Opts) -> - ForceCreate = maps:get(force_create, Opts, false), case lookup(InstId) of {ok, _} -> {ok, already_created}; - _ -> - Res0 = #{id => InstId, mod => ResourceType, config => Config, - status => starting, state => undefined}, - %% The `emqx_resource:call_start/3` need the instance exist beforehand - ets:insert(emqx_resource_instance, {InstId, Res0}), - case emqx_resource:call_start(InstId, ResourceType, Config) of - {ok, ResourceState} -> - ok = emqx_plugin_libs_metrics:create_metrics(resource_metrics, InstId), - %% this is the first time we do health check, this will update the - %% status and then do ets:insert/2 - _ = do_health_check(Res0#{state => ResourceState}), - HealthCheckInterval = maps:get(health_check_interval, Opts, 15000), - emqx_resource_health_check_sup:create_health_check_process(InstId, HealthCheckInterval), + {error, not_found} -> + case do_start(InstId, ResourceType, Config, Opts) of + ok -> + ok = emqx_resource_health_check_sup:create_checker(InstId, + maps:get(health_check_interval, Opts, 15000)), + ok = emqx_plugin_libs_metrics:clear_metrics(resource_metrics, InstId), {ok, force_lookup(InstId)}; - {error, Reason} when ForceCreate == true -> - logger:error("start ~ts resource ~ts failed: ~p, " - "force_create it as a stopped resource", - [ResourceType, InstId, Reason]), - ets:insert(emqx_resource_instance, {InstId, Res0}), - {ok, Res0}; - {error, Reason} when ForceCreate == false -> - ets:delete(emqx_resource_instance, InstId), - {error, Reason} + Error -> Error end end. -do_create_dry_run(InstId, ResourceType, Config) -> - case emqx_resource:call_start(InstId, ResourceType, Config) of - {ok, ResourceState0} -> - Return = case emqx_resource:call_health_check(InstId, ResourceType, ResourceState0) of - {ok, ResourceState1} -> ok; - {error, Reason, ResourceState1} -> - {error, Reason} - end, - _ = emqx_resource:call_stop(InstId, ResourceType, ResourceState1), +do_create_dry_run(ResourceType, Config) -> + InstId = make_test_id(), + Opts = #{force_create => false}, + case do_create(InstId, ResourceType, Config, Opts) of + {ok, Data} -> + Return = do_health_check(Data), + _ = do_remove(Data), Return; {error, Reason} -> {error, Reason} end. -do_remove(InstId) -> - case lookup(InstId) of - {ok, #{mod := Mod, state := ResourceState}} -> - do_remove(Mod, InstId, ResourceState); - Error -> - Error - end. +do_remove(Instance) -> + do_remove(Instance, true). -do_remove(Mod, InstId, ResourceState) -> - do_remove(Mod, InstId, ResourceState, true). - -do_remove(Mod, InstId, ResourceState, ClearMetrics) -> - _ = emqx_resource:call_stop(InstId, Mod, ResourceState), +do_remove(InstId, ClearMetrics) when is_binary(InstId) -> + do_with_instance_data(InstId, fun do_remove/2, [ClearMetrics]); +do_remove(#{id := InstId} = Data, ClearMetrics) -> + _ = do_stop(Data), ets:delete(emqx_resource_instance, InstId), case ClearMetrics of true -> ok = emqx_plugin_libs_metrics:clear_metrics(resource_metrics, InstId); false -> ok end, - _ = emqx_resource_health_check_sup:delete_health_check_process(InstId), + _ = emqx_resource_health_check_sup:delete_checker(InstId), ok. -do_restart(InstId) -> +do_restart(InstId, Opts) -> case lookup(InstId) of - {ok, #{mod := Mod, state := ResourceState, config := Config} = Data} -> - _ = case ResourceState of - undefined -> ok; - _ -> emqx_resource:call_stop(InstId, Mod, ResourceState) - end, - case emqx_resource:call_start(InstId, Mod, Config) of - {ok, NewResourceState} -> - ets:insert(emqx_resource_instance, - {InstId, Data#{state => NewResourceState, status => started}}), - ok; - {error, Reason} -> - ets:insert(emqx_resource_instance, {InstId, Data#{status => stopped}}), - {error, Reason} - end; + {ok, #{mod := ResourceType, config := Config} = Data} -> + ok = do_stop(Data), + do_start(InstId, ResourceType, Config, Opts); Error -> Error end. -do_stop(InstId) -> - case lookup(InstId) of - {ok, #{mod := Mod, state := ResourceState} = Data} -> - _ = emqx_resource:call_stop(InstId, Mod, ResourceState), - ets:insert(emqx_resource_instance, {InstId, Data#{status => stopped}}), +do_start(InstId, ResourceType, Config, Opts) when is_binary(InstId) -> + ForceCreate = maps:get(force_create, Opts, false), + Res0 = #{id => InstId, mod => ResourceType, config => Config, + status => starting, state => undefined}, + %% The `emqx_resource:call_start/3` need the instance exist beforehand + ets:insert(emqx_resource_instance, {InstId, Res0}), + case emqx_resource:call_start(InstId, ResourceType, Config) of + {ok, ResourceState} -> + %% this is the first time we do health check, this will update the + %% status and then do ets:insert/2 + _ = do_health_check(Res0#{state => ResourceState}), ok; - Error -> - Error + {error, Reason} when ForceCreate == true -> + logger:warning("start ~ts resource ~ts failed: ~p, force_create it", + [ResourceType, InstId, Reason]), + ets:insert(emqx_resource_instance, {InstId, Res0}), + ok; + {error, Reason} when ForceCreate == false -> + ets:delete(emqx_resource_instance, InstId), + {error, Reason} end. +do_stop(InstId) when is_binary(InstId) -> + do_with_instance_data(InstId, fun do_stop/1, []); +do_stop(#{state := undefined}) -> + ok; +do_stop(#{id := InstId, mod := Mod, state := ResourceState} = Data) -> + _ = emqx_resource:call_stop(InstId, Mod, ResourceState), + ets:insert(emqx_resource_instance, {InstId, Data#{status => stopped}}), + ok. + do_health_check(InstId) when is_binary(InstId) -> case lookup(InstId) of {ok, Data} -> do_health_check(Data); @@ -284,6 +267,12 @@ do_health_check(#{id := InstId, mod := Mod, state := ResourceState0} = Data) -> %% internal functions %%------------------------------------------------------------------------------ +do_with_instance_data(InstId, Do, Args) -> + case lookup(InstId) of + {ok, Data} -> erlang:apply(Do, [Data | Args]); + Error -> Error + end. + proc_name(Mod, Id) -> list_to_atom(lists:concat([Mod, "_", Id])). diff --git a/apps/emqx_resource/src/emqx_resource_sup.erl b/apps/emqx_resource/src/emqx_resource_sup.erl index b5655e301..99d601ec4 100644 --- a/apps/emqx_resource/src/emqx_resource_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_sup.erl @@ -49,7 +49,7 @@ init([]) -> #{id => emqx_resource_health_check_sup, start => {emqx_resource_health_check_sup, start_link, []}, restart => transient, - shutdown => 5000, type => supervisor, modules => [emqx_resource_health_check_sup]}, + shutdown => infinity, type => supervisor, modules => [emqx_resource_health_check_sup]}, {ok, {SupFlags, [HealthCheck, Metrics | ResourceInsts]}}. %% internal functions From e1ab331a30b2ef32289b9de255ebb8438dc03a3c Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sun, 2 Jan 2022 20:11:25 +0800 Subject: [PATCH 86/99] refactor(resource): support async create mode --- apps/emqx_authn/src/emqx_authn_api.erl | 2 +- apps/emqx_bridge/src/emqx_bridge.erl | 5 +- apps/emqx_resource/include/emqx_resource.hrl | 2 +- .../src/emqx_resource_instance.erl | 64 +++++++++++-------- 4 files changed, 42 insertions(+), 31 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 64ce3e6a4..a7e073581 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -979,7 +979,7 @@ authenticator_examples() -> mechanism => <<"password-based">>, backend => <<"http">>, method => <<"post">>, - url => <<"http://127.0.0.2:8080">>, + url => <<"http://127.0.0.1:18083">>, headers => #{ <<"content-type">> => <<"application/json">> }, diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 6e014f2ec..b7a74c9a4 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -218,7 +218,7 @@ create(Type, Name, Conf) -> ?SLOG(info, #{msg => "create bridge", type => Type, name => Name, config => Conf}), case emqx_resource:create_local(resource_id(Type, Name), emqx_bridge:resource_type(Type), - parse_confs(Type, Name, Conf), #{force_create => true}) of + parse_confs(Type, Name, Conf), #{async_create => true}) of {ok, already_created} -> maybe_disable_bridge(Type, Name, Conf); {ok, _} -> maybe_disable_bridge(Type, Name, Conf); {error, Reason} -> {error, Reason} @@ -263,7 +263,8 @@ recreate(Type, Name) -> recreate(Type, Name, Conf) -> emqx_resource:recreate_local(resource_id(Type, Name), - emqx_bridge:resource_type(Type), parse_confs(Type, Name, Conf), []). + emqx_bridge:resource_type(Type), parse_confs(Type, Name, Conf), + #{async_create => true}). create_dry_run(Type, Conf) -> Conf0 = Conf#{<<"ingress">> => #{<<"remote_topic">> => <<"t">>}}, diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index ed1de18cf..363e40a5f 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -33,7 +33,7 @@ %% The emqx_resource:create/4 will return OK event if the Mod:on_start/2 fails, %% the 'status' of the resource will be 'stopped' in this case. %% Defaults to 'false' - force_create => boolean() + async_create => boolean() }. -type after_query() :: {[OnSuccess :: after_query_fun()], [OnFailed :: after_query_fun()]} | undefined. diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index 2701b70d6..4ac72ca4d 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -148,14 +148,19 @@ code_change(_OldVsn, State, _Extra) -> do_recreate(InstId, ResourceType, NewConfig, Opts) -> case lookup(InstId) of - {ok, #{mod := ResourceType} = Data} -> + {ok, #{mod := ResourceType, status := started} = Data} -> + %% If this resource is in use (status='started'), we should make sure + %% the new config is OK before removing the old one. case do_create_dry_run(ResourceType, NewConfig) of ok -> do_remove(Data, false), - do_create(InstId, ResourceType, NewConfig, Opts#{force_create => true}); + do_create(InstId, ResourceType, NewConfig, Opts); Error -> Error end; + {ok, #{mod := ResourceType, status := _} = Data} -> + do_remove(Data, false), + do_create(InstId, ResourceType, NewConfig, Opts); {ok, #{mod := Mod}} when Mod =/= ResourceType -> {error, updating_to_incorrect_resource_type}; {error, not_found} -> @@ -164,21 +169,21 @@ do_recreate(InstId, ResourceType, NewConfig, Opts) -> do_create(InstId, ResourceType, Config, Opts) -> case lookup(InstId) of - {ok, _} -> {ok, already_created}; + {ok, _} -> + {ok, already_created}; {error, not_found} -> case do_start(InstId, ResourceType, Config, Opts) of ok -> - ok = emqx_resource_health_check_sup:create_checker(InstId, - maps:get(health_check_interval, Opts, 15000)), ok = emqx_plugin_libs_metrics:clear_metrics(resource_metrics, InstId), {ok, force_lookup(InstId)}; - Error -> Error + Error -> + Error end end. do_create_dry_run(ResourceType, Config) -> InstId = make_test_id(), - Opts = #{force_create => false}, + Opts = #{async_create => false}, case do_create(InstId, ResourceType, Config, Opts) of {ok, Data} -> Return = do_health_check(Data), @@ -200,7 +205,6 @@ do_remove(#{id := InstId} = Data, ClearMetrics) -> true -> ok = emqx_plugin_libs_metrics:clear_metrics(resource_metrics, InstId); false -> ok end, - _ = emqx_resource_health_check_sup:delete_checker(InstId), ok. do_restart(InstId, Opts) -> @@ -213,24 +217,32 @@ do_restart(InstId, Opts) -> end. do_start(InstId, ResourceType, Config, Opts) when is_binary(InstId) -> - ForceCreate = maps:get(force_create, Opts, false), - Res0 = #{id => InstId, mod => ResourceType, config => Config, - status => starting, state => undefined}, + InitData = #{id => InstId, mod => ResourceType, config => Config, + status => starting, state => undefined}, %% The `emqx_resource:call_start/3` need the instance exist beforehand - ets:insert(emqx_resource_instance, {InstId, Res0}), + ets:insert(emqx_resource_instance, {InstId, InitData}), + case maps:get(async_create, Opts, false) of + false -> + start_and_check(InstId, ResourceType, Config, Opts, InitData); + true -> + spawn(fun() -> + start_and_check(InstId, ResourceType, Config, Opts, InitData) + end), + ok + end. + +start_and_check(InstId, ResourceType, Config, Opts, Data) -> case emqx_resource:call_start(InstId, ResourceType, Config) of {ok, ResourceState} -> - %% this is the first time we do health check, this will update the - %% status and then do ets:insert/2 - _ = do_health_check(Res0#{state => ResourceState}), - ok; - {error, Reason} when ForceCreate == true -> - logger:warning("start ~ts resource ~ts failed: ~p, force_create it", - [ResourceType, InstId, Reason]), - ets:insert(emqx_resource_instance, {InstId, Res0}), - ok; - {error, Reason} when ForceCreate == false -> - ets:delete(emqx_resource_instance, InstId), + Data2 = Data#{state => ResourceState}, + ets:insert(emqx_resource_instance, {InstId, Data2}), + case maps:get(async_create, Opts, false) of + false -> do_health_check(Data2); + true -> emqx_resource_health_check_sup:create_checker(InstId, + maps:get(health_check_interval, Opts, 15000)) + end; + {error, Reason} -> + ets:insert(emqx_resource_instance, {InstId, Data#{status => stopped}}), {error, Reason} end. @@ -240,14 +252,12 @@ do_stop(#{state := undefined}) -> ok; do_stop(#{id := InstId, mod := Mod, state := ResourceState} = Data) -> _ = emqx_resource:call_stop(InstId, Mod, ResourceState), + ok = emqx_resource_health_check_sup:delete_checker(InstId), ets:insert(emqx_resource_instance, {InstId, Data#{status => stopped}}), ok. do_health_check(InstId) when is_binary(InstId) -> - case lookup(InstId) of - {ok, Data} -> do_health_check(Data); - Error -> Error - end; + do_with_instance_data(InstId, fun do_health_check/1, []); do_health_check(#{state := undefined}) -> {error, resource_not_initialized}; do_health_check(#{id := InstId, mod := Mod, state := ResourceState0} = Data) -> From a64b29ff76f9278d053b1264cf6bc48fd91407b5 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sun, 2 Jan 2022 20:32:24 +0800 Subject: [PATCH 87/99] fix(resource): re-create the helth checker if already exists --- .../src/emqx_resource_health_check.erl | 35 ++++++++++++++++++- .../src/emqx_resource_health_check_sup.erl | 32 +---------------- .../src/emqx_resource_instance.erl | 4 +-- 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_health_check.erl b/apps/emqx_resource/src/emqx_resource_health_check.erl index dc0795fb3..dfda683e1 100644 --- a/apps/emqx_resource/src/emqx_resource_health_check.erl +++ b/apps/emqx_resource/src/emqx_resource_health_check.erl @@ -15,14 +15,47 @@ %%-------------------------------------------------------------------- -module(emqx_resource_health_check). --export([start_link/2]). +-export([ start_link/2 + , create_checker/2 + , delete_checker/1 + ]). -export([health_check/2]). +-define(SUP, emqx_resource_health_check_sup). +-define(ID(NAME), {resource_health_check, NAME}). + +child_spec(Name, Sleep) -> + #{id => ?ID(Name), + start => {?MODULE, start_link, [Name, Sleep]}, + restart => transient, + shutdown => 5000, type => worker, modules => [?MODULE]}. + start_link(Name, Sleep) -> Pid = proc_lib:spawn_link(?MODULE, health_check, [Name, Sleep]), {ok, Pid}. +create_checker(Name, Sleep) -> + case supervisor:start_child(?SUP, child_spec(Name, Sleep)) of + {ok, _} -> ok; + {error, already_present} -> ok; + {error, {already_started, _}} -> + ok = delete_checker(Name), + create_checker(Name, Sleep); + Error -> Error + end. + +delete_checker(Name) -> + case supervisor:terminate_child(?SUP, {health_check, Name}) of + ok -> + case supervisor:delete_child(?SUP, {health_check, Name}) of + {error, not_found} -> ok; + Error -> Error + end; + {error, not_found} -> ok; + Error -> Error + end. + health_check(Name, SleepTime) -> timer:sleep(SleepTime), case emqx_resource:health_check(Name) of diff --git a/apps/emqx_resource/src/emqx_resource_health_check_sup.erl b/apps/emqx_resource/src/emqx_resource_health_check_sup.erl index 6a2b07e94..e17186114 100644 --- a/apps/emqx_resource/src/emqx_resource_health_check_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_health_check_sup.erl @@ -19,18 +19,7 @@ -export([start_link/0]). --export([init/1, - create_checker/2, - delete_checker/1]). - --define(HEALTH_CHECK_MOD, emqx_resource_health_check). --define(ID(NAME), {resource_health_check, NAME}). - -child_spec(Name, Sleep) -> - #{id => ?ID(Name), - start => {?HEALTH_CHECK_MOD, start_link, [Name, Sleep]}, - restart => transient, - shutdown => 5000, type => worker, modules => [?HEALTH_CHECK_MOD]}. +-export([init/1]). start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). @@ -38,22 +27,3 @@ start_link() -> init([]) -> SupFlags = #{strategy => one_for_one, intensity => 10, period => 10}, {ok, {SupFlags, []}}. - -create_checker(Name, Sleep) -> - case supervisor:start_child(?MODULE, child_spec(Name, Sleep)) of - {ok, _} -> ok; - {error, already_present} -> ok; - {error, {already_started, _}} -> ok; - Error -> Error - end. - -delete_checker(Name) -> - case supervisor:terminate_child(?MODULE, {health_check, Name}) of - ok -> - case supervisor:delete_child(?MODULE, {health_check, Name}) of - {error, not_found} -> ok; - Error -> Error - end; - {error, not_found} -> ok; - Error -> Error - end. diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index 4ac72ca4d..a847e85f5 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -238,7 +238,7 @@ start_and_check(InstId, ResourceType, Config, Opts, Data) -> ets:insert(emqx_resource_instance, {InstId, Data2}), case maps:get(async_create, Opts, false) of false -> do_health_check(Data2); - true -> emqx_resource_health_check_sup:create_checker(InstId, + true -> emqx_resource_health_check:create_checker(InstId, maps:get(health_check_interval, Opts, 15000)) end; {error, Reason} -> @@ -252,7 +252,7 @@ do_stop(#{state := undefined}) -> ok; do_stop(#{id := InstId, mod := Mod, state := ResourceState} = Data) -> _ = emqx_resource:call_stop(InstId, Mod, ResourceState), - ok = emqx_resource_health_check_sup:delete_checker(InstId), + ok = emqx_resource_health_check:delete_checker(InstId), ets:insert(emqx_resource_instance, {InstId, Data#{status => stopped}}), ok. From 11736dc1d7b8b211c2256ad11e3fad3d5b12c07a Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sun, 2 Jan 2022 22:45:32 +0800 Subject: [PATCH 88/99] fix(bridge): check health immediately after updated --- apps/emqx_bridge/src/emqx_bridge.erl | 2 +- .../src/emqx_resource_health_check.erl | 18 ++++++------- .../src/emqx_resource_instance.erl | 27 ++++++++++--------- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index b7a74c9a4..786b99c44 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -247,7 +247,7 @@ update(Type, Name, {OldConf, Conf}) -> ?SLOG(warning, #{ msg => "updating_a_non-exist_bridge_need_create_a_new_one" , type => Type, name => Name, config => Conf}), create(Type, Name, Conf); - {error, Reason} -> {update_bridge_failed, Reason} + {error, Reason} -> {error, {update_bridge_failed, Reason}} end; true -> %% we don't need to recreate the bridge if this config change is only to diff --git a/apps/emqx_resource/src/emqx_resource_health_check.erl b/apps/emqx_resource/src/emqx_resource_health_check.erl index dfda683e1..032ff6999 100644 --- a/apps/emqx_resource/src/emqx_resource_health_check.erl +++ b/apps/emqx_resource/src/emqx_resource_health_check.erl @@ -36,28 +36,25 @@ start_link(Name, Sleep) -> {ok, Pid}. create_checker(Name, Sleep) -> + create_checker(Name, Sleep, false). + +create_checker(Name, Sleep, Retry) -> case supervisor:start_child(?SUP, child_spec(Name, Sleep)) of {ok, _} -> ok; {error, already_present} -> ok; - {error, {already_started, _}} -> + {error, {already_started, _}} when Retry == false -> ok = delete_checker(Name), - create_checker(Name, Sleep); + create_checker(Name, Sleep, true); Error -> Error end. delete_checker(Name) -> - case supervisor:terminate_child(?SUP, {health_check, Name}) of - ok -> - case supervisor:delete_child(?SUP, {health_check, Name}) of - {error, not_found} -> ok; - Error -> Error - end; - {error, not_found} -> ok; + case supervisor:terminate_child(?SUP, ?ID(Name)) of + ok -> supervisor:delete_child(?SUP, ?ID(Name)); Error -> Error end. health_check(Name, SleepTime) -> - timer:sleep(SleepTime), case emqx_resource:health_check(Name) of ok -> emqx_alarm:deactivate(Name); @@ -65,4 +62,5 @@ health_check(Name, SleepTime) -> emqx_alarm:activate(Name, #{name => Name}, <>) end, + timer:sleep(SleepTime), health_check(Name, SleepTime). diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index a847e85f5..86318e355 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -140,11 +140,14 @@ code_change(_OldVsn, State, _Extra) -> %%------------------------------------------------------------------------------ %% suppress the race condition check, as these functions are protected in gproc workers --dialyzer({nowarn_function, [do_recreate/4, - do_create/4, - do_restart/2, - do_stop/1, - do_health_check/1]}). +-dialyzer({nowarn_function, [ do_recreate/4 + , do_create/4 + , do_restart/2 + , do_start/4 + , do_stop/1 + , do_health_check/1 + , start_and_check/5 + ]}). do_recreate(InstId, ResourceType, NewConfig, Opts) -> case lookup(InstId) of @@ -183,12 +186,12 @@ do_create(InstId, ResourceType, Config, Opts) -> do_create_dry_run(ResourceType, Config) -> InstId = make_test_id(), - Opts = #{async_create => false}, - case do_create(InstId, ResourceType, Config, Opts) of - {ok, Data} -> - Return = do_health_check(Data), - _ = do_remove(Data), - Return; + case emqx_resource:call_start(InstId, ResourceType, Config) of + {ok, ResourceState} -> + case emqx_resource:call_health_check(InstId, ResourceType, ResourceState) of + {ok, _} -> ok; + {error, Reason, _} -> {error, Reason} + end; {error, Reason} -> {error, Reason} end. @@ -252,7 +255,7 @@ do_stop(#{state := undefined}) -> ok; do_stop(#{id := InstId, mod := Mod, state := ResourceState} = Data) -> _ = emqx_resource:call_stop(InstId, Mod, ResourceState), - ok = emqx_resource_health_check:delete_checker(InstId), + _ = emqx_resource_health_check:delete_checker(InstId), ets:insert(emqx_resource_instance, {InstId, Data#{status => stopped}}), ok. From e95445728c1570ec05783fd39281ab5746574f2d Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sun, 2 Jan 2022 23:46:58 +0800 Subject: [PATCH 89/99] fix(test): wait until the bridge ready --- apps/emqx_bridge/src/emqx_bridge.erl | 5 ++++ .../test/emqx_bridge_api_SUITE.erl | 13 +++++++++++ .../test/emqx_connector_api_SUITE.erl | 23 ++++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 786b99c44..f27603bf9 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -35,6 +35,7 @@ ]). -export([ load/0 + , lookup/1 , lookup/2 , lookup/3 , list/0 @@ -191,6 +192,10 @@ list_bridges_by_connector(ConnectorId) -> [B || B = #{raw_config := #{<<"connector">> := Id}} <- list(), ConnectorId =:= Id]. +lookup(Id) -> + {Type, Name} = parse_bridge_id(Id), + lookup(Type, Name). + lookup(Type, Name) -> RawConf = emqx:get_raw_config([bridges, Type, Name], #{}), lookup(Type, Name, RawConf). diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 807ad32f6..16da7395f 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -160,6 +160,7 @@ t_http_crud_apis(_) -> } = jsx:decode(Bridge), %% send an message to emqx and the message should be forwarded to the HTTP server + wait_for_resource_ready(BridgeID, 5), Body = <<"my msg">>, emqx:publish(emqx_message:make(<<"emqx_http/1">>, Body)), ?assert( @@ -212,6 +213,7 @@ t_http_crud_apis(_) -> }, jsx:decode(Bridge3Str)), %% send an message to emqx again, check the path has been changed + wait_for_resource_ready(BridgeID, 5), emqx:publish(emqx_message:make(<<"emqx_http/1">>, Body)), ?assert( receive @@ -320,3 +322,14 @@ auth_header_() -> operation_path(Oper, BridgeID) -> uri(["bridges", BridgeID, "operation", Oper]). + +wait_for_resource_ready(InstId, 0) -> + ct:pal("--- bridge ~p: ~p", [InstId, emqx_bridge:lookup(InstId)]), + ct:fail(wait_resource_timeout); +wait_for_resource_ready(InstId, Retry) -> + case emqx_bridge:lookup(InstId) of + {ok, #{resource_data := #{status := started}}} -> ok; + _ -> + timer:sleep(100), + wait_for_resource_ready(InstId, Retry-1) + end. diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index 4caf700a9..12a3a8e23 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -241,6 +241,7 @@ t_mqtt_conn_bridge_ingress(_) -> emqx:subscribe(LocalTopic), %% PUBLISH a message to the 'remote' broker, as we have only one broker, %% the remote broker is also the local one. + wait_for_resource_ready(BridgeIDIngress, 5), emqx:publish(emqx_message:make(RemoteTopic, Payload)), %% we should receive a message on the local broker, with specified topic ?assert( @@ -309,6 +310,7 @@ t_mqtt_conn_bridge_egress(_) -> emqx:subscribe(RemoteTopic), %% PUBLISH a message to the 'local' broker, as we have only one broker, %% the remote broker is also the local one. + wait_for_resource_ready(BridgeIDEgress, 5), emqx:publish(emqx_message:make(LocalTopic, Payload)), %% we should receive a message on the "remote" broker, with specified topic @@ -370,6 +372,7 @@ t_mqtt_conn_update(_) -> , <<"status">> := <<"connected">> , <<"connector">> := ConnctorID } = jsx:decode(Bridge), + wait_for_resource_ready(BridgeIDEgress, 2), %% then we try to update 'server' of the connector, to an unavailable IP address %% the update should fail because of 'unreachable' or 'connrefused' @@ -412,6 +415,11 @@ t_mqtt_conn_update2(_) -> , <<"status">> := <<"disconnected">> , <<"connector">> := ConnctorID } = jsx:decode(Bridge), + %% We try to fix the 'server' parameter, to another unavailable server.. + %% The update should success: we don't check the connectivity of the new config + %% if the resource is now disconnected. + {ok, 200, _} = request(put, uri(["connectors", ConnctorID]), + ?MQTT_CONNECOTR2(<<"127.0.0.1:2604">>)), %% we fix the 'server' parameter to a normal one, it should work {ok, 200, _} = request(put, uri(["connectors", ConnctorID]), ?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)), @@ -444,9 +452,9 @@ t_mqtt_conn_update3(_) -> <<"name">> => ?BRIDGE_NAME_EGRESS }), #{ <<"id">> := BridgeIDEgress - , <<"status">> := <<"connected">> , <<"connector">> := ConnctorID } = jsx:decode(Bridge), + wait_for_resource_ready(BridgeIDEgress, 2), %% delete the connector should fail because it is in use by a bridge {ok, 403, _} = request(delete, uri(["connectors", ConnctorID]), []), @@ -499,6 +507,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> emqx:subscribe(LocalTopic), %% PUBLISH a message to the 'remote' broker, as we have only one broker, %% the remote broker is also the local one. + wait_for_resource_ready(BridgeIDIngress, 5), emqx:publish(emqx_message:make(RemoteTopic, Payload)), %% we should receive a message on the local broker, with specified topic ?assert( @@ -563,6 +572,7 @@ t_egress_mqtt_bridge_with_rules(_) -> emqx:subscribe(RemoteTopic), %% PUBLISH a message to the 'local' broker, as we have only one broker, %% the remote broker is also the local one. + wait_for_resource_ready(BridgeIDEgress, 5), emqx:publish(emqx_message:make(LocalTopic, Payload)), %% we should receive a message on the "remote" broker, with specified topic ?assert( @@ -583,6 +593,7 @@ t_egress_mqtt_bridge_with_rules(_) -> RuleTopic = <<"t/1">>, RemoteTopic2 = <<"remote_topic/", RuleTopic/binary>>, emqx:subscribe(RemoteTopic2), + wait_for_resource_ready(BridgeIDEgress, 5), emqx:publish(emqx_message:make(RuleTopic, Payload2)), {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []), #{ <<"id">> := RuleId @@ -646,3 +657,13 @@ auth_header_() -> {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password), {"Authorization", "Bearer " ++ binary_to_list(Token)}. +wait_for_resource_ready(InstId, 0) -> + ct:pal("--- bridge ~p: ~p", [InstId, emqx_bridge:lookup(InstId)]), + ct:fail(wait_resource_timeout); +wait_for_resource_ready(InstId, Retry) -> + case emqx_bridge:lookup(InstId) of + {ok, #{resource_data := #{status := started}}} -> ok; + _ -> + timer:sleep(100), + wait_for_resource_ready(InstId, Retry-1) + end. From 65b1ce80d97ae74d5e4da44043547366cdb11301 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 3 Jan 2022 11:47:43 +0100 Subject: [PATCH 90/99] chore: fix deps discrepancy --- apps/emqx_dashboard/rebar.config | 2 +- apps/emqx_prometheus/rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_dashboard/rebar.config b/apps/emqx_dashboard/rebar.config index 618fc203d..bd765fa4b 100644 --- a/apps/emqx_dashboard/rebar.config +++ b/apps/emqx_dashboard/rebar.config @@ -1,4 +1,4 @@ -{deps, [ {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}} +{deps, [ {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.6"}}} , {emqx, {path, "../emqx"}} ]}. diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index a12e3092d..9cd506995 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -2,7 +2,7 @@ [ %% FIXME: tag this as v3.1.3 {prometheus, {git, "https://github.com/emqx/prometheus.erl", {ref, "9994c76adca40d91a2545102230ccce2423fd8a7"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.2"}}}, - {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.7"}}} + {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.9"}}} ]}. {edoc_opts, [{preprocess, true}]}. From 2fbe2dd0c3ab2913e3f237baeaa9686e774e3105 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 3 Jan 2022 12:02:51 +0100 Subject: [PATCH 91/99] fix(boot): ensure emqx_conf is the first app to boot The first one to boot after emqx_machine --- apps/emqx_machine/src/emqx_machine_boot.erl | 4 ++-- apps/emqx_machine/test/emqx_machine_tests.erl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 9d45f791e..93bd31f21 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -85,7 +85,6 @@ reboot_apps() -> , esockd , ranch , cowboy - , emqx_conf , emqx , emqx_prometheus , emqx_modules @@ -121,7 +120,8 @@ sorted_reboot_apps(Apps) -> NoDepApps = add_apps_to_digraph(G, Apps), case digraph_utils:topsort(G) of Sorted when is_list(Sorted) -> - Sorted ++ (NoDepApps -- Sorted); + %% ensure emqx_conf boot up first + [emqx_conf | Sorted ++ (NoDepApps -- Sorted)]; false -> Loops = find_loops(G), error({circular_application_dependency, Loops}) diff --git a/apps/emqx_machine/test/emqx_machine_tests.erl b/apps/emqx_machine/test/emqx_machine_tests.erl index 1a562b815..074167f95 100644 --- a/apps/emqx_machine/test/emqx_machine_tests.erl +++ b/apps/emqx_machine/test/emqx_machine_tests.erl @@ -38,7 +38,7 @@ sorted_reboot_apps_cycle_test() -> check_order(Apps) -> AllApps = lists:usort(lists:append([[A | Deps] || {A, Deps} <- Apps])), - Sorted = emqx_machine_boot:sorted_reboot_apps(Apps), + [emqx_conf | Sorted] = emqx_machine_boot:sorted_reboot_apps(Apps), case length(AllApps) =:= length(Sorted) of true -> ok; false -> error({AllApps, Sorted}) From 57a466f372cd761ec35e441ce96261185d945095 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 3 Jan 2022 11:55:38 +0100 Subject: [PATCH 92/99] chore: sync rebar dependency to mix build --- mix.exs | 8 ++++---- mix.lock | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mix.exs b/mix.exs index 363b40f4b..44f565177 100644 --- a/mix.exs +++ b/mix.exs @@ -45,7 +45,7 @@ defmodule EMQXUmbrella.MixProject do # other exact versions, and not ranges. [ {:lc, github: "qzhuyan/lc", tag: "0.1.2"}, - {:typerefl, github: "k32/typerefl", tag: "0.8.5", override: true}, + {:typerefl, github: "k32/typerefl", tag: "0.8.6", override: true}, {:ehttpc, github: "emqx/ehttpc", tag: "0.1.12"}, {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, @@ -54,8 +54,8 @@ defmodule EMQXUmbrella.MixProject do {:mria, github: "emqx/mria", tag: "0.1.5", override: true}, {:ekka, github: "emqx/ekka", tag: "0.11.2", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.5.1", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.2.7", override: true}, - {:ecpool, github: "emqx/ecpool", tag: "0.5.1"}, + {:minirest, github: "emqx/minirest", tag: "1.2.9", override: true}, + {:ecpool, github: "emqx/ecpool", tag: "0.5.2"}, {:replayq, "0.3.3", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, {:emqtt, github: "emqx/emqtt", tag: "1.4.3", override: true}, @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by ehttpc and emqtt {:gun, github: "emqx/gun", tag: "1.3.6", override: true}, # in conflict by emqx_connectior and system_monitor - {:epgsql, github: "epgsql/epgsql", tag: "4.6.0", override: true}, + {:epgsql, github: "emqx/epgsql", tag: "4.7-emqx.1", override: true}, # in conflict by mongodb and eredis_cluster {:poolboy, github: "emqx/poolboy", tag: "1.5.2", override: true}, # in conflict by emqx and observer_cli diff --git a/mix.lock b/mix.lock index 460eef4a7..4a8338530 100644 --- a/mix.lock +++ b/mix.lock @@ -4,14 +4,14 @@ "cowboy": {:git, "https://github.com/emqx/cowboy.git", "e3ed6c2ab3ac29988d26ed1f176def47b6e8d6de", [tag: "2.9.0"]}, "cowboy_swagger": {:git, "https://github.com/inaka/cowboy_swagger", "bc441df7988da0f5c5d11ae0861c394dc30995c5", [tag: "2.5.0"]}, "cowlib": {:git, "https://github.com/ninenines/cowlib.git", "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", [ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363"]}, - "ecpool": {:git, "https://github.com/emqx/ecpool.git", "0516d2cebd14654ef8c583c347e4a0b01363b86d", [tag: "0.5.1"]}, + "ecpool": {:git, "https://github.com/emqx/ecpool.git", "8cd58f169239b96f8aa9a6cfdb5fa0275038ed1b", [tag: "0.5.2"]}, "eetcd": {:git, "https://github.com/zhongwencool/eetcd", "69d50aca98247953ee8a3ff58423a693f8318d90", [tag: "v0.3.4"]}, "ehttpc": {:git, "https://github.com/emqx/ehttpc.git", "7b1a76b2353b385725e62f948cd399c7040467f8", [tag: "0.1.12"]}, "ekka": {:git, "https://github.com/emqx/ekka.git", "70f2250e5e968e0c1da64e5b4733c5eb0eb402de", [tag: "0.11.2"]}, "eldap2": {:git, "https://github.com/emqx/eldap2", "f595f67b094db3b9dc07941337706621e815431f", [tag: "v0.2.2"]}, "emqtt": {:git, "https://github.com/emqx/emqtt.git", "25892ef48a979a9dfbd74d86133cb28cf11f3cf4", [tag: "1.4.3"]}, "emqx_http_lib": {:git, "https://github.com/emqx/emqx_http_lib.git", "b84d42239fb09fecf50d9469fac914fb9b8efe34", [tag: "0.4.1"]}, - "epgsql": {:git, "https://github.com/epgsql/epgsql.git", "f7530f63ae40ea2b81bae7d4a33292212349b761", [tag: "4.6.0"]}, + "epgsql": {:git, "https://github.com/emqx/epgsql.git", "102e53c53ed5b14d37c5428cbf91530e01944369", [tag: "4.7-emqx.1"]}, "eredis": {:git, "https://github.com/emqx/eredis", "75f2b8eedbe631136326680225efbcd2684e93e7", [tag: "1.2.5"]}, "eredis_cluster": {:git, "https://github.com/emqx/eredis_cluster", "624749b4aef25668e9c7a545427fdc663a04faef", [tag: "0.6.7"]}, "esasl": {:git, "https://github.com/emqx/esasl.git", "96d7ac9f6c156017dd35b30df2dd722ae469c7f0", [tag: "0.2.0"]}, @@ -29,9 +29,9 @@ "jose": {:git, "https://github.com/potatosalad/erlang-jose.git", "991649695aaccd92c8effb1c1e88e6159fe8e9a6", [tag: "1.11.2"]}, "jsx": {:git, "https://github.com/talentdeficit/jsx.git", "bb9b3e570a7efe331eed0900c3a5188043a850d7", [tag: "v3.1.0"]}, "lc": {:git, "https://github.com/qzhuyan/lc.git", "6f98d098e5aaf4fcd6afbbb2acca96855c474600", [tag: "0.1.2"]}, - "minirest": {:git, "https://github.com/emqx/minirest.git", "f3f80b3e07295d8b6db22ed456318e0cc9dd167f", [tag: "1.2.7"]}, + "minirest": {:git, "https://github.com/emqx/minirest.git", "a620032ae1971a8b7eafa352ea214f6ab24f70dd", [tag: "1.2.9"]}, "mnesia_rocksdb": {:git, "https://github.com/k32/mnesia_rocksdb", "68a80d127c49005480e0dd1f73149e8621052100", [tag: "0.1.5-k32"]}, - "mongodb": {:git, "https://github.com/emqx/mongodb-erlang", "2ffe62f42dafb98eaafead9d340a674c5f9279a5", [tag: "v3.0.10"]}, + "mongodb": {:git, "https://github.com/emqx/mongodb-erlang", "7c6d6fcd562c0b9c88e123ee1cf5ed0b97d0f3ac", [tag: "v3.0.11"]}, "mria": {:git, "https://github.com/emqx/mria.git", "2bf3a71abc3635f910be4b943fa4ccbf8b8257fa", [tag: "0.1.5"]}, "mysql": {:git, "https://github.com/emqx/mysql-otp", "bdabac44cc8836a9e23897b7e1b77c7df7e04f70", [tag: "1.7.1"]}, "observer_cli": {:hex, :observer_cli, "1.7.1", "c9ca1f623a3ef0158283a3c37cd7b7235bfe85927ad6e26396dd247e2057f5a1", [:mix, :rebar3], [{:recon, "~>2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "4ccafaaa2ce01b85ddd14591f4d5f6731b4e13b610a70fb841f0701178478280"}, @@ -50,5 +50,5 @@ "supervisor3": {:hex, :supervisor3, "1.1.9", "f1a3cc12fb6197526f548e79c9fe2b4af0c74efb8a687917b3b1ebe5e9c9368d", [:rebar3], [], "hexpm", "71b177c08f8cab9ec8ecb81c1aa28a23bbc24aac4b468c2db69840229d78d5c4"}, "system_monitor": {:git, "https://github.com/k32/system_monitor.git", "3b4b381bf9503695cd764ecf22067ac6542cee89", [tag: "2.2.1"]}, "trails": {:hex, :trails, "2.3.0", "b09703f056705f4943e14fff077b98c711a6f48fad40f4ff0b350794074ad69c", [:rebar3], [{:cowboy, "2.8.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:ranch, "2.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "40804001eb80417aa9d02400f39b7216956c3f251539a8a6096a69b3fac0ea07"}, - "typerefl": {:git, "https://github.com/k32/typerefl.git", "0cafafe1a6ce94f8709f237e890026a290a3e36f", [tag: "0.8.5"]}, + "typerefl": {:git, "https://github.com/k32/typerefl.git", "ccb1256b747587d829bd5bd8477875aa6929e4d6", [tag: "0.8.6"]}, } From f866488bc27c062f72d202b1ea52a5c1831b984e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 3 Jan 2022 14:34:58 +0100 Subject: [PATCH 93/99] test: fix mongodb tls suite dryruns return error when local health check fails --- apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl index e62f895a2..c3b04ec41 100644 --- a/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl @@ -91,7 +91,7 @@ t_create_invalid_server_name(_Config) -> create_mongo_auth_with_ssl_opts( #{<<"server_name_indication">> => <<"authn-server-unknown-host">>, <<"verify">> => <<"verify_peer">>}), - fun({ok, _}, Trace) -> + fun({error, _}, Trace) -> ?assertEqual( [failed], ?projection( @@ -109,7 +109,7 @@ t_create_invalid_version(_Config) -> #{<<"server_name_indication">> => <<"authn-server">>, <<"verify">> => <<"verify_peer">>, <<"versions">> => [<<"tlsv1.1">>]}), - fun({ok, _}, Trace) -> + fun({error, _}, Trace) -> ?assertEqual( [failed], ?projection( @@ -118,7 +118,7 @@ t_create_invalid_version(_Config) -> end). -%% docker-compose-mongo-single-tls.yaml: +%% docker-compose-mongo-single-tls.yaml: %% --setParameter opensslCipherConfig='HIGH:!EXPORT:!aNULL:!DHE:!kDHE@STRENGTH' t_invalid_ciphers(_Config) -> @@ -128,7 +128,7 @@ t_invalid_ciphers(_Config) -> <<"verify">> => <<"verify_peer">>, <<"versions">> => [<<"tlsv1.2">>], <<"ciphers">> => [<<"DHE-RSA-AES256-GCM-SHA384">>]}), - fun({ok, _}, Trace) -> + fun({error, _}, Trace) -> ?assertEqual( [failed], ?projection( From 6c99b64e4c2bda18495f0bfb9f0058c86231817a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 4 Jan 2022 00:24:11 +0100 Subject: [PATCH 94/99] refactor(emqx_plugin_libs_pool): structured logging --- .../emqx_connector/src/emqx_connector_mongo.erl | 16 +++++++++++----- .../src/emqx_plugin_libs_pool.erl | 17 ++++++++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 4efb92088..10dd6ae6e 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -34,6 +34,8 @@ , on_jsonify/1 ]). + +%% ecpool callback -export([connect/1]). -export([roots/0, fields/1]). @@ -125,7 +127,7 @@ on_start(InstId, Config = #{mongo_type := Type, {options, init_topology_options(maps:to_list(Topology), [])}, {worker_options, init_worker_options(maps:to_list(NConfig), SslOpts)}], PoolName = emqx_plugin_libs_pool:pool_name(InstId), - _ = emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts), + ok = emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts), {ok, #{poolname => PoolName, type => Type}}. on_stop(InstId, #{poolname := PoolName}) -> @@ -177,18 +179,22 @@ health_check(PoolName) -> %% =================================================================== -check_worker_health(Worker) -> +%% TODO: log reasons +check_worker_health(Worker) -> case ecpool_worker:client(Worker) of {ok, Conn} -> %% we don't care if this returns something or not, we just to test the connection try mongo_api:find_one(Conn, <<"foo">>, #{}, #{}) of - {error, _} -> false; + {error, _Reason} -> + false; _ -> true catch - _Class:_Error -> false + _ : _ -> + false end; - _ -> false + _ -> + false end. connect(Opts) -> diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl index 03c5bdc8f..c067fb6fe 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl @@ -22,26 +22,33 @@ , health_check/3 ]). +-include_lib("emqx/include/logger.hrl"). + pool_name(ID) when is_binary(ID) -> list_to_atom(binary_to_list(ID)). start_pool(Name, Mod, Options) -> case ecpool:start_sup_pool(Name, Mod, Options) of - {ok, _} -> logger:log(info, "Initiated ~0p Successfully", [Name]); + {ok, _} -> + ?SLOG(info, #{msg => "start_ecpool_ok", pool_name => Name}); {error, {already_started, _Pid}} -> stop_pool(Name), start_pool(Name, Mod, Options); {error, Reason} -> - logger:log(error, "Initiate ~0p failed ~0p", [Name, Reason]), + ?SLOG(error, #{msg => "start_ecpool_error", pool_name => Name, + reason => Reason}), error({start_pool_failed, Name}) end. stop_pool(Name) -> case ecpool:stop_sup_pool(Name) of - ok -> logger:log(info, "Destroyed ~0p Successfully", [Name]); - {error, not_found} -> ok; + ok -> + ?SLOG(info, #{msg => "stop_ecpool_ok", pool_name => Name}); + {error, not_found} -> + ok; {error, Reason} -> - logger:log(error, "Destroy ~0p failed, ~0p", [Name, Reason]), + ?SLOG(error, #{msg => "stop_ecpool_failed", pool_name => Name, + reason => Reason}), error({stop_pool_failed, Name}) end. From 7b478817bda46d266b37abe4787a7b9ae63a2875 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 4 Jan 2022 00:25:09 +0100 Subject: [PATCH 95/99] test: test authz against 'single' mogodb --- apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl | 11 +++++++++-- apps/emqx_resource/src/emqx_resource_instance.erl | 1 - 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index a623ccf79..95d084a86 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -38,8 +38,8 @@ }). -define(SOURCE2, #{<<"type">> => <<"mongodb">>, <<"enable">> => true, - <<"mongo_type">> => <<"sharded">>, - <<"servers">> => <<"127.0.0.1:27017,192.168.0.1:27017">>, + <<"mongo_type">> => <<"single">>, + <<"server">> => <<"127.0.0.1:27017">>, <<"pool_size">> => 1, <<"database">> => <<"mqtt">>, <<"ssl">> => #{<<"enable">> => false}, @@ -160,6 +160,13 @@ end_per_testcase(_, _Config) -> ok. %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ +t_mongodb_connectivity(_) -> + Type = single, + Hosts = ["127.0.0.1:27017", "192.168.0.1:27017"], + TopologyOpts = [{pool_size, 1}], + WorkerOpts = [{database, <<"mqtt">>}, {ssl, false}], + {ok, Pid} = mongo_api:connect(Type, Hosts, TopologyOpts, WorkerOpts), + ?assertEqual(undefined, mongo_api:find_one(Pid, <<"foo">>, #{<<"key">> => 123}, #{})). t_api(_) -> {ok, 200, Result1} = request(get, uri(["authorization", "sources"]), []), diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index 86318e355..201738272 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -26,7 +26,6 @@ -export([ lookup/1 , get_metrics/1 , list_all/0 - , make_test_id/0 ]). -export([ hash_call/2 From e3c8f67aaf3bc11fa44a75e5d3c24bd7ffbe1ef0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 4 Jan 2022 00:43:29 +0100 Subject: [PATCH 96/99] test: resource creation now checks health adjust test case to work with new code behavior --- apps/emqx_authz/test/emqx_authz_http_SUITE.erl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl index 2187fba73..9a3b86958 100644 --- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -343,17 +343,16 @@ t_create_replace(_Config) -> listener => {tcp, default} }, - %% Bad URL + %% Create with valid URL ok = setup_handler_and_config( fun(Req0, State) -> Req = cowboy_req:reply(200, Req0), {ok, Req, State} end, - #{<<"base_url">> => <<"http://127.0.0.1:33331/authz">>}), - + #{<<"base_url">> => <<"http://127.0.0.1:33333/authz">>}), ?assertEqual( - deny, + allow, emqx_access_control:authorize(ClientInfo, publish, <<"t">>)), %% Changing to other bad config does not work @@ -366,14 +365,14 @@ t_create_replace(_Config) -> emqx_authz:update({?CMD_REPLACE, http}, BadConfig)), ?assertEqual( - deny, + allow, emqx_access_control:authorize(ClientInfo, publish, <<"t">>)), %% Changing to valid config OkConfig = maps:merge( raw_http_authz_config(), #{<<"base_url">> => <<"http://127.0.0.1:33333/authz">>}), - + ?assertMatch( {ok, _}, emqx_authz:update({?CMD_REPLACE, http}, OkConfig)), From 1eaac9ea12d18383f8eaf720c1b650105fde1e67 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 4 Jan 2022 00:14:40 +0100 Subject: [PATCH 97/99] feat: add redbug --- mix.exs | 2 ++ mix.lock | 1 + rebar.config | 1 + rebar.config.erl | 1 + 4 files changed, 5 insertions(+) diff --git a/mix.exs b/mix.exs index 44f565177..b4b9ef701 100644 --- a/mix.exs +++ b/mix.exs @@ -45,6 +45,7 @@ defmodule EMQXUmbrella.MixProject do # other exact versions, and not ranges. [ {:lc, github: "qzhuyan/lc", tag: "0.1.2"}, + {:redbug, "2.0.7"}, {:typerefl, github: "k32/typerefl", tag: "0.8.6", override: true}, {:ehttpc, github: "emqx/ehttpc", tag: "0.1.12"}, {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, @@ -163,6 +164,7 @@ defmodule EMQXUmbrella.MixProject do inets: :permanent, compiler: :permanent, runtime_tools: :permanent, + redbug: :permanent, hocon: :load, emqx: :load, emqx_conf: :load, diff --git a/mix.lock b/mix.lock index 4a8338530..c8a70a321 100644 --- a/mix.lock +++ b/mix.lock @@ -41,6 +41,7 @@ "quicer": {:git, "https://github.com/emqx/quic.git", "ef73617d0f10f0f30f3aa77eb4a2f6ae071a2e29", [tag: "0.0.9"]}, "ranch": {:git, "https://github.com/ninenines/ranch.git", "a692f44567034dacf5efcaa24a24183788594eb7", [ref: "a692f44567034dacf5efcaa24a24183788594eb7"]}, "recon": {:git, "https://github.com/ferd/recon.git", "f7b6c08e6e9e2219db58bfb012c58c178822e01e", [tag: "2.5.1"]}, + "redbug": {:hex, :redbug, "2.0.7", "40f477681e4957a1ebce19b23a9b53cd8113da6133014c506e3e194a6e4ffc89", [:rebar3], [], "hexpm", "3624feb7a4b78fd9ae0e66cc3158fe7422770ad6987a1ebf8df4d3303b1c4b0c"}, "replayq": {:hex, :replayq, "0.3.3", "29344e4fd7c41c232d7f20d7a6e6712169ca585583bbb4bb6dd518f04e0d6cc4", [:rebar3], [], "hexpm", "3a527aff0960cf7ba7d189c79d7f0fbc170adb62e351acc223ccd6d094095c27"}, "rocksdb": {:git, "https://github.com/k32/erlang-rocksdb.git", "e74972c3da4fe1f08eb66d39fce643a2d25a60be", [tag: "1.7.2-k32"]}, "rulesql": {:git, "https://github.com/emqx/rulesql.git", "fec11b1a3cbf98480d19c06d3aca10442e1e02a9", [tag: "0.1.4"]}, diff --git a/rebar.config b/rebar.config index 59be4e6c0..17d4a0ce3 100644 --- a/rebar.config +++ b/rebar.config @@ -45,6 +45,7 @@ {deps, [ {lc, {git, "https://github.com/qzhuyan/lc.git", {tag, "0.1.2"}}} + , {redbug, "2.0.7"} , {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.6"}}} , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.12"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 982f5c3b9..dc92d85ee 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -250,6 +250,7 @@ relx_apps(ReleaseType, Edition) -> , inets , compiler , runtime_tools + , redbug , {hocon, load} , {emqx, load} % started by emqx_machine , {emqx_conf, load} From b5e7344752796e0a220ea3e485f43072f21bd9dc Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 4 Jan 2022 01:00:09 +0100 Subject: [PATCH 98/99] build(emqx_plugin_libs): add rebar.config to help mix build --- apps/emqx_plugin_libs/rebar.config | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 apps/emqx_plugin_libs/rebar.config diff --git a/apps/emqx_plugin_libs/rebar.config b/apps/emqx_plugin_libs/rebar.config new file mode 100644 index 000000000..07646091a --- /dev/null +++ b/apps/emqx_plugin_libs/rebar.config @@ -0,0 +1,2 @@ +{deps, [ {emqx, {path, "../emqx"}} + ]}. From 503bf54c77a2fe44736f0e979edaafd516fff0e6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 4 Jan 2022 08:18:46 +0100 Subject: [PATCH 99/99] test(authz): fix meck calls f --- .github/workflows/build_slim_packages.yaml | 2 +- .../test/emqx_authz_api_sources_SUITE.erl | 21 +++++++------------ 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index aaa56b30b..a0b1d5b55 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -60,7 +60,7 @@ jobs: - macos-11 - macos-10.15 - runs-on: ${{ matrix.macos }} + runs-on: ${{ matrix.macos }} steps: - uses: actions/checkout@v2 diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index 95d084a86..123b6b3ac 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -26,6 +26,7 @@ -define(HOST, "http://127.0.0.1:18083/"). -define(API_VERSION, "v5"). -define(BASE_PATH, "api"). +-define(MONGO_SINGLE_HOST, "mongo:27017"). -define(SOURCE1, #{<<"type">> => <<"http">>, <<"enable">> => true, @@ -39,7 +40,7 @@ -define(SOURCE2, #{<<"type">> => <<"mongodb">>, <<"enable">> => true, <<"mongo_type">> => <<"single">>, - <<"server">> => <<"127.0.0.1:27017">>, + <<"server">> => <>, <<"pool_size">> => 1, <<"database">> => <<"mqtt">>, <<"ssl">> => #{<<"enable">> => false}, @@ -48,7 +49,7 @@ }). -define(SOURCE3, #{<<"type">> => <<"mysql">>, <<"enable">> => true, - <<"server">> => <<"127.0.0.1:3306">>, + <<"server">> => <<"mysql:3306">>, <<"pool_size">> => 1, <<"database">> => <<"mqtt">>, <<"username">> => <<"xx">>, @@ -59,7 +60,7 @@ }). -define(SOURCE4, #{<<"type">> => <<"postgresql">>, <<"enable">> => true, - <<"server">> => <<"127.0.0.1:5432">>, + <<"server">> => <<"pgsql:5432">>, <<"pool_size">> => 1, <<"database">> => <<"mqtt">>, <<"username">> => <<"xx">>, @@ -70,7 +71,7 @@ }). -define(SOURCE5, #{<<"type">> => <<"redis">>, <<"enable">> => true, - <<"servers">> => <<"127.0.0.1:6379, 127.0.0.1:6380">>, + <<"servers">> => <<"redis:6379,127.0.0.1:6380">>, <<"pool_size">> => 1, <<"database">> => 0, <<"password">> => <<"ee">>, @@ -96,12 +97,13 @@ groups() -> init_per_suite(Config) -> meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end), + meck:expect(emqx_resource, create_local, fun(_, _, _) -> {ok, meck_data} end), meck:expect(emqx_resource, create_dry_run_local, fun(emqx_connector_mysql, _) -> ok; + (emqx_connector_mongo, _) -> ok; (T, C) -> meck:passthrough([T, C]) end), - meck:expect(emqx_resource, health_check, fun(_) -> ok end), + meck:expect(emqx_resource, health_check, fun(St) -> {ok, St} end), meck:expect(emqx_resource, remove_local, fun(_) -> ok end ), ok = emqx_common_test_helpers:start_apps( @@ -160,13 +162,6 @@ end_per_testcase(_, _Config) -> ok. %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ -t_mongodb_connectivity(_) -> - Type = single, - Hosts = ["127.0.0.1:27017", "192.168.0.1:27017"], - TopologyOpts = [{pool_size, 1}], - WorkerOpts = [{database, <<"mqtt">>}, {ssl, false}], - {ok, Pid} = mongo_api:connect(Type, Hosts, TopologyOpts, WorkerOpts), - ?assertEqual(undefined, mongo_api:find_one(Pid, <<"foo">>, #{<<"key">> => 123}, #{})). t_api(_) -> {ok, 200, Result1} = request(get, uri(["authorization", "sources"]), []),