From 45c4fb33a8df1fe74d766b5524cd74dcece0c507 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 7 Dec 2023 16:06:04 +0800 Subject: [PATCH 01/50] fix: occasionally return stale view when updating configurations on different nodes concurrently --- apps/emqx_conf/src/emqx_cluster_rpc.erl | 52 +++++--- .../emqx_conf/test/emqx_cluster_rpc_SUITE.erl | 114 ++++++++++++++++-- changes/ce/fix-12121.en.md | 1 + 3 files changed, 137 insertions(+), 30 deletions(-) create mode 100644 changes/ce/fix-12121.en.md diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 756a5ec30..373bafdba 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -343,13 +343,8 @@ handle_call(reset, _From, State) -> _ = mria:clear_table(?CLUSTER_COMMIT), _ = mria:clear_table(?CLUSTER_MFA), {reply, ok, State, {continue, ?CATCH_UP}}; -handle_call(?INITIATE(MFA), _From, State = #{node := Node}) -> - case transaction(fun ?MODULE:init_mfa/2, [Node, MFA]) of - {atomic, {ok, TnxId, Result}} -> - {reply, {ok, TnxId, Result}, State, {continue, ?CATCH_UP}}; - {aborted, Error} -> - {reply, {init_failure, Error}, State, {continue, ?CATCH_UP}} - end; +handle_call(?INITIATE(MFA), _From, State) -> + do_initiate(MFA, State, 1, #{}); handle_call(skip_failed_commit, _From, State = #{node := Node}) -> Timeout = catch_up(State, true), {atomic, LatestId} = transaction(fun ?MODULE:get_node_tnx_id/1, [Node]), @@ -465,11 +460,40 @@ get_oldest_mfa_id() -> Id -> Id end. +do_initiate(_MFA, State, Count, Failure) when Count > 10 -> + %% refuse to initiate cluster call from this node + %% because it's likely that the caller is based on + %% a stale view event we retry 10 time. + Error = stale_view_of_cluster_msg(Failure, Count), + {reply, {init_failure, Error}, State, {continue, ?CATCH_UP}}; +do_initiate(MFA, State = #{node := Node}, Count, Failure0) -> + case transaction(fun ?MODULE:init_mfa/2, [Node, MFA]) of + {atomic, {ok, TnxId, Result}} -> + {reply, {ok, TnxId, Result}, State, {continue, ?CATCH_UP}}; + {atomic, {retry, Failure1}} when Failure0 =:= Failure1 -> + %% Useless retry, so we return early. + Error = stale_view_of_cluster_msg(Failure0, Count), + {reply, {init_failure, Error}, State, {continue, ?CATCH_UP}}; + {atomic, {retry, Failure1}} -> + catch_up(State), + do_initiate(MFA, State, Count + 1, Failure1); + {aborted, Error} -> + {reply, {init_failure, Error}, State, {continue, ?CATCH_UP}} + end. + +stale_view_of_cluster_msg(Meta, Count) -> + Reason = Meta#{ + msg => stale_view_of_cluster_state, + retry_times => Count + }, + ?SLOG(warning, Reason), + Reason. + %% The entry point of a config change transaction. init_mfa(Node, MFA) -> mnesia:write_lock_table(?CLUSTER_MFA), LatestId = get_cluster_tnx_id(), - MyTnxId = get_node_tnx_id(node()), + MyTnxId = get_node_tnx_id(Node), case MyTnxId =:= LatestId of true -> TnxId = LatestId + 1, @@ -486,16 +510,8 @@ init_mfa(Node, MFA) -> {false, Error} -> mnesia:abort(Error) end; false -> - %% refuse to initiate cluster call from this node - %% because it's likely that the caller is based on - %% a stale view. - Reason = #{ - msg => stale_view_of_cluster_state, - cluster_tnx_id => LatestId, - node_tnx_id => MyTnxId - }, - ?SLOG(warning, Reason), - mnesia:abort({error, Reason}) + Meta = #{cluster_tnx_id => LatestId, node_tnx_id => MyTnxId}, + {retry, Meta} end. transaction(Func, Args) -> diff --git a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl index 8cdfcaeea..99a0766ec 100644 --- a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl +++ b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl @@ -35,9 +35,10 @@ all() -> t_commit_ok_apply_fail_on_other_node_then_recover, t_del_stale_mfa, t_skip_failed_commit, - t_fast_forward_commit + t_fast_forward_commit, + t_commit_concurrency ]. -suite() -> [{timetrap, {minutes, 3}}]. +suite() -> [{timetrap, {minutes, 5}}]. groups() -> []. init_per_suite(Config) -> @@ -63,6 +64,7 @@ end_per_suite(_Config) -> ok. init_per_testcase(_TestCase, Config) -> + stop(), start(), Config. @@ -119,17 +121,101 @@ t_commit_crash_test(_Config) -> t_commit_ok_but_apply_fail_on_other_node(_Config) -> emqx_cluster_rpc:reset(), {atomic, []} = emqx_cluster_rpc:status(), - MFA = {M, F, A} = {?MODULE, failed_on_node, [erlang:whereis(?NODE1)]}, + Pid = self(), + {BaseM, BaseF, BaseA} = {?MODULE, echo, [Pid, test]}, + {ok, _TnxId, ok} = multicall(BaseM, BaseF, BaseA), + ?assertEqual(ok, receive_msg(3, test)), + + {M, F, A} = {?MODULE, failed_on_node, [erlang:whereis(?NODE1)]}, {ok, _, ok} = multicall(M, F, A, 1, 1000), - {atomic, [Status]} = emqx_cluster_rpc:status(), - ?assertEqual(MFA, maps:get(mfa, Status)), - ?assertEqual(node(), maps:get(node, Status)), + {atomic, AllStatus} = emqx_cluster_rpc:status(), + Node = node(), + ?assertEqual( + [ + {1, {Node, emqx_cluster_rpc2}}, + {1, {Node, emqx_cluster_rpc3}}, + {2, Node} + ], + lists:sort([{T, N} || #{tnx_id := T, node := N} <- AllStatus]) + ), erlang:send(?NODE2, test), Call = emqx_cluster_rpc:make_initiate_call_req(M, F, A), - Res = gen_server:call(?NODE2, Call), - ?assertEqual({init_failure, "MFA return not ok"}, Res), + Res1 = gen_server:call(?NODE2, Call), + Res2 = gen_server:call(?NODE3, Call), + %% Node2 is retry on tnx_id 1, and should not run Next MFA. + ?assertEqual( + {init_failure, #{ + msg => stale_view_of_cluster_state, + retry_times => 2, + cluster_tnx_id => 2, + node_tnx_id => 1 + }}, + Res1 + ), + ?assertEqual(Res1, Res2), ok. +t_commit_concurrency(_Config) -> + emqx_cluster_rpc:reset(), + {atomic, []} = emqx_cluster_rpc:status(), + Pid = self(), + {BaseM, BaseF, BaseA} = {?MODULE, echo, [Pid, test]}, + {ok, _TnxId, ok} = multicall(BaseM, BaseF, BaseA), + ?assertEqual(ok, receive_msg(3, test)), + + %% call concurrently without stale tnx_id error + Workers = lists:seq(1, 256), + lists:foreach( + fun(Seq) -> + {EchoM, EchoF, EchoA} = {?MODULE, echo_delay, [Pid, Seq]}, + Call = emqx_cluster_rpc:make_initiate_call_req(EchoM, EchoF, EchoA), + spawn_link(fun() -> + ?assertMatch({ok, _, ok}, gen_server:call(?NODE1, Call, infinity)) + end), + spawn_link(fun() -> + ?assertMatch({ok, _, ok}, gen_server:call(?NODE2, Call, infinity)) + end), + spawn_link(fun() -> + ?assertMatch({ok, _, ok}, gen_server:call(?NODE3, Call, infinity)) + end) + end, + Workers + ), + %% receive seq msg in order + List = lists:sort(receive_seq_msg([])), + ?assertEqual(256 * 3 * 3, length(List), List), + {atomic, Status} = emqx_cluster_rpc:status(), + lists:map( + fun(#{tnx_id := TnxId} = S) -> + ?assertEqual(256 * 3 + 1, TnxId, S) + end, + Status + ), + AllMsgIndex = lists:flatten(lists:duplicate(9, Workers)), + Result = + lists:foldl( + fun(Index, Acc) -> + ?assertEqual(true, lists:keymember(Index, 1, Acc), {Index, Acc}), + lists:keydelete(Index, 1, Acc) + end, + List, + AllMsgIndex + ), + ?assertEqual([], Result), + receive + Unknown -> throw({receive_unknown_msg, Unknown}) + after 1000 -> ok + end, + ok. + +receive_seq_msg(Acc) -> + receive + {msg, Seq, Time, Pid} -> + receive_seq_msg([{Seq, Time, Pid} | Acc]) + after 3000 -> + Acc + end. + t_catch_up_status_handle_next_commit(_Config) -> emqx_cluster_rpc:reset(), {atomic, []} = emqx_cluster_rpc:status(), @@ -296,9 +382,8 @@ stop() -> erlang:exit(P, kill) end end - || N <- [?NODE1, ?NODE2, ?NODE3] - ], - gen_server:stop(emqx_cluster_rpc_cleaner, normal, 5000). + || N <- [?NODE1, ?NODE2, ?NODE3, emqx_cluster_rpc_cleaner] + ]. receive_msg(0, _Msg) -> ok; @@ -306,7 +391,7 @@ receive_msg(Count, Msg) when Count > 0 -> receive Msg -> receive_msg(Count - 1, Msg) - after 800 -> + after 1000 -> timeout end. @@ -314,6 +399,11 @@ echo(Pid, Msg) -> erlang:send(Pid, Msg), ok. +echo_delay(Pid, Msg) -> + timer:sleep(rand:uniform(150)), + erlang:send(Pid, {msg, Msg, erlang:system_time(), self()}), + ok. + failed_on_node(Pid) -> case Pid =:= self() of true -> ok; diff --git a/changes/ce/fix-12121.en.md b/changes/ce/fix-12121.en.md new file mode 100644 index 000000000..3f0782826 --- /dev/null +++ b/changes/ce/fix-12121.en.md @@ -0,0 +1 @@ +Fixed occasionally return stale view when updating configurations on different nodes concurrently From e57d796df5213d6487865001ed476b738ba3ac65 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 8 Dec 2023 12:47:32 +0800 Subject: [PATCH 02/50] fix(ldap): Adjust logger level for authentication failures --- apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl index 1f2af261e..26dd4daac 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl @@ -52,7 +52,7 @@ authenticate( {ok, #{result := ok}} -> {ok, #{is_superuser => false}}; {ok, #{result := 'invalidCredentials'}} -> - ?TRACE_AUTHN_PROVIDER(error, "ldap_bind_failed", #{ + ?TRACE_AUTHN_PROVIDER(info, "ldap_bind_failed", #{ resource => ResourceId, reason => 'invalidCredentials' }), From e5cf9e40ac61f82ad367ea050c90c8ad17bff3eb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 8 Dec 2023 08:25:20 +0100 Subject: [PATCH 03/50] refactor: update default enterprise license --- apps/emqx_license/src/emqx_license_schema.erl | 10 ++++------ rel/i18n/emqx_license_schema.hocon | 8 ++++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/apps/emqx_license/src/emqx_license_schema.erl b/apps/emqx_license/src/emqx_license_schema.erl index aeced0ab0..7c8df0878 100644 --- a/apps/emqx_license/src/emqx_license_schema.erl +++ b/apps/emqx_license/src/emqx_license_schema.erl @@ -86,14 +86,12 @@ check_license_watermark(Conf) -> end. %% @doc The default license key. -%% This default license has 1000 connections limit. -%% It is issued on 2023-01-09 and valid for 5 years (1825 days) +%% This default license has 25 connections limit. +%% Issued on 2023-12-08 and valid for 5 years (1825 days) %% NOTE: when updating a new key, the schema doc in emqx_license_schema.hocon %% should be updated accordingly default_license() -> << - "MjIwMTExCjAKMTAKRXZhbHVhdGlvbgpjb250YWN0QGVtcXguaW8KZ" - "GVmYXVsdAoyMDIzMDEwOQoxODI1CjEwMAo=.MEUCIG62t8W15g05f" - "1cKx3tA3YgJoR0dmyHOPCdbUxBGxgKKAiEAhHKh8dUwhU+OxNEaOn" - "8mgRDtiT3R8RZooqy6dEsOmDI=" + "MjIwMTExCjAKMTAKRXZhbHVhdGlvbgpjb250YWN0QGVtcXguaW8KdHJpYWwKMjAyMzEyMDgKMTgyNQoyNQo=." + "MEUCIE271MtH+4bb39OZKD4mvVkurwZ3LX44KUvuOxkbjQz2AiEAqL7BP44PMUS5z5SAN1M4y3v3h47J8qORAqcuetnyexw=" >>. diff --git a/rel/i18n/emqx_license_schema.hocon b/rel/i18n/emqx_license_schema.hocon index e280af257..ab565a465 100644 --- a/rel/i18n/emqx_license_schema.hocon +++ b/rel/i18n/emqx_license_schema.hocon @@ -42,11 +42,11 @@ key_field.label: license_root.desc: """Defines the EMQX Enterprise license. +EMQX Enterprise is initially provided with a default trial license. +This license, issued in December 2023, is valid for a period of 5 years. +It supports up to 25 concurrent connections, catering to early-stage development and testing needs. -The default license has 100 connections limit, it is issued on 2023-01-09 and valid for 5 years (1825 days). - -EMQX comes with a default trial license. For production use, please -visit https://www.emqx.com/apply-licenses/emqx to apply.""" +For deploying EMQX Enterprise in a production environment, a different license is required. You can apply for a production license by visiting https://www.emqx.com/apply-licenses/emqx""" license_root.label: """License""" From ddc984becd01aa5765884013b6ed6064d2b2e41b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 8 Dec 2023 08:29:29 +0100 Subject: [PATCH 04/50] docs: add changelog for license renewal --- changes/ee/feat-12129.en.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ee/feat-12129.en.md diff --git a/changes/ee/feat-12129.en.md b/changes/ee/feat-12129.en.md new file mode 100644 index 000000000..378237a51 --- /dev/null +++ b/changes/ee/feat-12129.en.md @@ -0,0 +1,4 @@ +Default license renewal. + +Replaced old license issued in Jan 2023. +New license supports up to 25 concurrent connections. From 41679808fed3b1235e05a36be9751711b3fe40d7 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 8 Dec 2023 08:35:58 +0100 Subject: [PATCH 05/50] chore: add version number in license apply link --- apps/emqx_license/include/emqx_license.hrl | 4 ++-- rel/i18n/emqx_license_schema.hocon | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_license/include/emqx_license.hrl b/apps/emqx_license/include/emqx_license.hrl index 60febc4cd..ccbe67cd1 100644 --- a/apps/emqx_license/include/emqx_license.hrl +++ b/apps/emqx_license/include/emqx_license.hrl @@ -11,7 +11,7 @@ "\n" "========================================================================\n" "Using an evaluation license limited to ~p concurrent connections.\n" - "Visit https://emqx.com/apply-licenses/emqx to apply a new license.\n" + "Visit https://emqx.com/apply-licenses/emqx?version=5 to apply a new license.\n" "Or contact EMQ customer services via email contact@emqx.io\n" "========================================================================\n" ). @@ -20,7 +20,7 @@ "\n" "========================================================================\n" "License has been expired for ~p days.\n" - "Visit https://emqx.com/apply-licenses/emqx to apply a new license.\n" + "Visit https://emqx.com/apply-licenses/emqx?version=5 to apply a new license.\n" "Or contact EMQ customer services via email contact@emqx.io\n" "========================================================================\n" ). diff --git a/rel/i18n/emqx_license_schema.hocon b/rel/i18n/emqx_license_schema.hocon index ab565a465..72f31266b 100644 --- a/rel/i18n/emqx_license_schema.hocon +++ b/rel/i18n/emqx_license_schema.hocon @@ -46,7 +46,7 @@ EMQX Enterprise is initially provided with a default trial license. This license, issued in December 2023, is valid for a period of 5 years. It supports up to 25 concurrent connections, catering to early-stage development and testing needs. -For deploying EMQX Enterprise in a production environment, a different license is required. You can apply for a production license by visiting https://www.emqx.com/apply-licenses/emqx""" +For deploying EMQX Enterprise in a production environment, a different license is required. You can apply for a production license by visiting https://www.emqx.com/apply-licenses/emqx?version=5""" license_root.label: """License""" From 0c80c7455498bd80b9c01b6f424326f3db4362ec Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Fri, 8 Dec 2023 09:59:19 +0100 Subject: [PATCH 06/50] fix: remove unnecessary prepare_statement from pgsql action --- apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl index 50c322941..d366a2690 100644 --- a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl +++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl @@ -64,8 +64,7 @@ fields(action_parameters) -> binary(), #{desc => ?DESC("sql_template"), default => default_sql(), format => <<"sql">>} )} - ] ++ - emqx_connector_schema_lib:prepare_statement_fields(); + ]; fields(pgsql_action) -> emqx_bridge_v2_schema:make_producer_action_schema( hoconsc:mk( From f4f45cf634016abaa954ccdc422ffc303197ae7c Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 8 Dec 2023 19:26:29 +0800 Subject: [PATCH 07/50] fix(authn_ldap): Improve the type inference of method union --- .../src/emqx_authn_ldap_schema.erl | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl index 3190d6e14..9d36867cd 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl @@ -55,7 +55,7 @@ fields(ldap) -> [ {method, ?HOCON( - hoconsc:union([?R_REF(hash_method), ?R_REF(bind_method)]), + hoconsc:union(fun method_union_member_selector/1), #{desc => ?DESC(method)} )} ]; @@ -88,6 +88,26 @@ desc(bind_method) -> desc(_) -> undefined. +method_union_member_selector(all_union_members) -> + [?R_REF(hash_method), ?R_REF(bind_method)]; +method_union_member_selector({value, Val}) -> + Val2 = + case is_map(Val) of + true -> emqx_utils_maps:binary_key_map(Val); + false -> Val + end, + case Val2 of + #{<<"type">> := <<"bind">>} -> + [?R_REF(bind_method)]; + #{<<"type">> := <<"hash">>} -> + [?R_REF(hash_method)]; + _ -> + throw(#{ + field_name => method, + expected => [bind_method, hash_method] + }) + end. + method_type(Type) -> ?HOCON(?ENUM([Type]), #{desc => ?DESC(?FUNCTION_NAME), default => Type}). From 1a8b59045b8439360f0a4a4300b8bdf3819e7f3c Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 8 Dec 2023 20:55:20 +0800 Subject: [PATCH 08/50] fix(gbt): Add unimplemented command call --- .../src/emqx_gbt32960_channel.erl | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl index 5cb65f104..65a696a16 100644 --- a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl +++ b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl @@ -51,6 +51,8 @@ inflight :: emqx_inflight:inflight(), %% Message Queue mqueue :: queue:queue(), + %% Subscriptions + subscriptions :: map(), retx_interval, retx_max_times, max_mqueue_len @@ -115,7 +117,7 @@ stats(#channel{inflight = Inflight, mqueue = Queue}) -> %% XXX: A fake stats for managed by emqx_management [ {subscriptions_cnt, 1}, - {subscriptions_max, 0}, + {subscriptions_max, 1}, {inflight_cnt, emqx_inflight:size(Inflight)}, {inflight_max, emqx_inflight:max_size(Inflight)}, {mqueue_len, queue:len(Queue)}, @@ -181,6 +183,7 @@ init( clientinfo = ClientInfo, inflight = emqx_inflight:new(1), mqueue = queue:new(), + subscriptions = #{}, timers = #{}, conn_state = idle, retx_interval = RetxInterv, @@ -370,9 +373,15 @@ handle_call(kick, _From, Channel) -> disconnect_and_shutdown(kicked, ok, Channel1); handle_call(discard, _From, Channel) -> disconnect_and_shutdown(discarded, ok, Channel); +handle_call({subscribe, _Topic, _SubOpts}, _From, Channel) -> + reply({error, not_support}, Channel); +handle_call({unsubscribe, _Topic}, _From, Channel) -> + reply({error, not_found}, Channel); +handle_call(subscriptions, _From, Channel = #channel{subscriptions = Subscriptions}) -> + reply({ok, maps:to_list(Subscriptions)}, Channel); handle_call(Req, _From, Channel) -> log(error, #{msg => "unexpected_call", call => Req}, Channel), - reply(ignored, Channel). + reply({error, unexpected_call}, Channel). %%-------------------------------------------------------------------- %% Handle cast @@ -606,8 +615,8 @@ process_connect( ) of {ok, #{session := Session}} -> - NChannel = Channel#channel{session = Session}, - subscribe_downlink(?DEFAULT_DOWNLINK_TOPIC, Channel), + NChannel0 = Channel#channel{session = Session}, + NChannel = subscribe_downlink(?DEFAULT_DOWNLINK_TOPIC, NChannel0), _ = upstreaming(Frame, NChannel), %% XXX: connection_accepted is not defined by stomp protocol _ = run_hooks(Ctx, 'client.connack', [ConnInfo, connection_accepted, #{}]), @@ -854,11 +863,13 @@ subscribe_downlink( #{ clientid := ClientId, mountpoint := Mountpoint - } - } + }, + subscriptions = Subscriptions + } = Channel ) -> {ParsedTopic, SubOpts0} = emqx_topic:parse(Topic), SubOpts = maps:merge(emqx_gateway_utils:default_subopts(), SubOpts0), MountedTopic = emqx_mountpoint:mount(Mountpoint, ParsedTopic), _ = emqx_broker:subscribe(MountedTopic, ClientId, SubOpts), - run_hooks(Ctx, 'session.subscribed', [ClientInfo, MountedTopic, SubOpts]). + run_hooks(Ctx, 'session.subscribed', [ClientInfo, MountedTopic, SubOpts]), + Channel#channel{subscriptions = Subscriptions#{MountedTopic => SubOpts}}. From 71607aa2ad5288adddd2ed61f35b7f987ab50def Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 28 Nov 2023 14:14:48 +0100 Subject: [PATCH 09/50] feat(emqx_bridge_mysql): port to shared connectors --- .../src/emqx_auth_mysql.app.src | 2 +- .../src/emqx_authn_mysql_schema.erl | 3 +- .../src/emqx_authz_mysql_schema.erl | 1 + apps/emqx_bridge/src/emqx_action_info.erl | 1 + .../src/schema/emqx_bridge_v2_schema.erl | 51 +++++- .../src/emqx_bridge_mongodb.erl | 4 +- .../src/emqx_bridge_mongodb_action_info.erl | 19 +-- .../src/emqx_bridge_mysql.app.src | 2 +- .../src/emqx_bridge_mysql.erl | 105 +++++++++++- .../src/emqx_bridge_mysql_action_info.erl | 64 ++++++++ .../src/emqx_bridge_mysql_connector.erl | 150 ++++++++++++++++++ .../test/emqx_bridge_mysql_SUITE.erl | 86 +++++++--- .../src/schema/emqx_connector_ee_schema.erl | 60 ++++--- .../src/schema/emqx_connector_schema.erl | 52 +++++- apps/emqx_mysql/rebar.config | 2 +- apps/emqx_mysql/src/emqx_mysql.erl | 61 +++++-- apps/emqx_utils/src/emqx_utils_maps.erl | 45 ++++-- .../emqx_utils/test/emqx_utils_maps_tests.erl | 43 +++++ mix.exs | 2 +- rebar.config | 2 +- rel/i18n/emqx_bridge_mysql.hocon | 10 ++ 21 files changed, 655 insertions(+), 110 deletions(-) create mode 100644 apps/emqx_bridge_mysql/src/emqx_bridge_mysql_action_info.erl create mode 100644 apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src index 38750b79a..24fdd2648 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_mysql, [ {description, "EMQX MySQL Authentication and Authorization"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {mod, {emqx_auth_mysql_app, []}}, {applications, [ diff --git a/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl b/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl index 6472794fe..df3b89ae9 100644 --- a/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl +++ b/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl @@ -55,8 +55,7 @@ fields(mysql) -> {password_hash_algorithm, fun emqx_authn_password_hashing:type_ro/1}, {query, fun query/1}, {query_timeout, fun query_timeout/1} - ] ++ emqx_authn_schema:common_fields() ++ - proplists:delete(prepare_statement, emqx_mysql:fields(config)). + ] ++ emqx_authn_schema:common_fields() ++ emqx_mysql:fields(config). desc(mysql) -> ?DESC(mysql); diff --git a/apps/emqx_auth_mysql/src/emqx_authz_mysql_schema.erl b/apps/emqx_auth_mysql/src/emqx_authz_mysql_schema.erl index 43f6ca6fa..c1cff7533 100644 --- a/apps/emqx_auth_mysql/src/emqx_authz_mysql_schema.erl +++ b/apps/emqx_auth_mysql/src/emqx_authz_mysql_schema.erl @@ -37,6 +37,7 @@ type() -> ?AUTHZ_TYPE. fields(mysql) -> emqx_authz_schema:authz_common_fields(?AUTHZ_TYPE) ++ emqx_mysql:fields(config) ++ + emqx_connector_schema_lib:prepare_statement_fields() ++ [{query, query()}]. desc(mysql) -> diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index f975a1c93..9376eeef3 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -79,6 +79,7 @@ hard_coded_action_info_modules_ee() -> emqx_bridge_kafka_action_info, emqx_bridge_matrix_action_info, emqx_bridge_mongodb_action_info, + emqx_bridge_mysql_action_info, emqx_bridge_pgsql_action_info, emqx_bridge_syskeeper_action_info, emqx_bridge_timescale_action_info, diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index 7cd6b5bd8..b271f4259 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -31,7 +31,8 @@ get_response/0, put_request/0, post_request/0, - examples/1 + examples/1, + action_values/4 ]). %% Exported for mocking @@ -103,6 +104,54 @@ bridge_api_union(Refs) -> end end. +-type http_method() :: get | post | put. +-type schema_example_map() :: #{atom() => term()}. + +-spec action_values(http_method(), atom(), atom(), schema_example_map()) -> schema_example_map(). +action_values(Method, ActionType, ConnectorType, ActionValues) -> + ActionTypeBin = atom_to_binary(ActionType), + ConnectorTypeBin = atom_to_binary(ConnectorType), + lists:foldl( + fun(M1, M2) -> + maps:merge(M1, M2) + end, + #{ + enable => true, + description => <<"My example ", ActionTypeBin/binary, " action">>, + connector => <>, + resource_opts => #{ + health_check_interval => "30s" + } + }, + [ + ActionValues, + method_values(Method, ActionType) + ] + ). + +-spec method_values(http_method(), atom()) -> schema_example_map(). +method_values(post, Type) -> + TypeBin = atom_to_binary(Type), + #{ + name => <>, + type => TypeBin + }; +method_values(get, Type) -> + maps:merge( + method_values(post, Type), + #{ + status => <<"connected">>, + node_status => [ + #{ + node => <<"emqx@localhost">>, + status => <<"connected">> + } + ] + } + ); +method_values(put, _Type) -> + #{}. + %%====================================================================================== %% HOCON Schema Callbacks %%====================================================================================== diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl index bc5e2eb74..61d3cec61 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl @@ -36,9 +36,7 @@ namespace() -> "bridge_mongodb". -roots() -> - %% ??? - []. +roots() -> []. fields("config") -> [ diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl index 823418f28..9fa19add2 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl @@ -31,9 +31,9 @@ connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) -> maps:merge( maps:without( [<<"connector">>], - map_unindent(<<"parameters">>, ActionConfig) + emqx_utils_maps:unindent(<<"parameters">>, ActionConfig) ), - map_unindent(<<"parameters">>, ConnectorConfig) + emqx_utils_maps:unindent(<<"parameters">>, ConnectorConfig) ) ). @@ -66,7 +66,7 @@ bridge_v1_config_to_connector_config(BridgeV1Config) -> make_config_map(PickKeys, IndentKeys, Config) -> Conf0 = maps:with(PickKeys, Config), - map_indent(<<"parameters">>, IndentKeys, Conf0). + emqx_utils_maps:indent(<<"parameters">>, IndentKeys, Conf0). bridge_v1_type_name() -> {fun ?MODULE:bridge_v1_type_name_fun/1, bridge_v1_type_names()}. @@ -86,18 +86,5 @@ v1_type(<<"rs">>) -> mongodb_rs; v1_type(<<"sharded">>) -> mongodb_sharded; v1_type(<<"single">>) -> mongodb_single. -map_unindent(Key, Map) -> - maps:merge( - maps:get(Key, Map), - maps:remove(Key, Map) - ). - -map_indent(IndentKey, PickKeys, Map) -> - maps:put( - IndentKey, - maps:with(PickKeys, Map), - maps:without(PickKeys, Map) - ). - schema_keys(Name) -> [bin(Key) || Key <- proplists:get_keys(?SCHEMA_MODULE:fields(Name))]. diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src index b1d110d36..50f535ac6 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src @@ -9,7 +9,7 @@ emqx_resource, emqx_mysql ]}, - {env, []}, + {env, [{emqx_action_info_modules, [emqx_bridge_mysql_action_info]}]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl index a72da7a1e..05c782b96 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl @@ -10,7 +10,9 @@ -import(hoconsc, [mk/2, enum/1, ref/2]). -export([ - conn_bridge_examples/1 + bridge_v2_examples/1, + conn_bridge_examples/1, + connector_examples/1 ]). -export([ @@ -20,6 +22,9 @@ desc/1 ]). +-define(CONNECTOR_TYPE, mysql). +-define(ACTION_TYPE, ?CONNECTOR_TYPE). + -define(DEFAULT_SQL, << "insert into t_mqtt_msg(msgid, topic, qos, payload, arrived) " "values (${id}, ${topic}, ${qos}, ${payload}, FROM_UNIXTIME(${timestamp}/1000))" @@ -28,6 +33,22 @@ %% ------------------------------------------------------------------------------------------------- %% api +bridge_v2_examples(Method) -> + [ + #{ + <<"mysql">> => + #{ + summary => <<"MySQL Action">>, + value => emqx_bridge_v2_schema:action_values( + Method, ?ACTION_TYPE, ?CONNECTOR_TYPE, action_values() + ) + } + } + ]. + +action_values() -> + #{parameters => #{sql => ?DEFAULT_SQL}}. + conn_bridge_examples(Method) -> [ #{ @@ -38,6 +59,29 @@ conn_bridge_examples(Method) -> } ]. +connector_examples(Method) -> + [ + #{ + <<"mysql">> => + #{ + summary => <<"MySQL Connector">>, + value => emqx_connector_schema:connector_values( + Method, ?CONNECTOR_TYPE, connector_values() + ) + } + } + ]. + +connector_values() -> + #{ + server => <<"127.0.0.1:3306">>, + database => <<"test">>, + pool_size => 8, + username => <<"root">>, + password => <<"******">>, + resource_opts => #{health_check_interval => <<"20s">>} + }. + values(_Method) -> #{ enable => true, @@ -80,17 +124,70 @@ fields("config") -> #{desc => ?DESC("local_topic"), default => undefined} )} ] ++ emqx_resource_schema:fields("resource_opts") ++ - (emqx_mysql:fields(config) -- - emqx_connector_schema_lib:prepare_statement_fields()); + emqx_mysql:fields(config); +fields(action) -> + {mysql, + mk( + hoconsc:map(name, ref(?MODULE, mysql_action)), + #{desc => <<"MySQL Action Config">>, required => false} + )}; +fields(mysql_action) -> + emqx_bridge_v2_schema:make_producer_action_schema( + mk( + ref(?MODULE, action_parameters), + #{ + required => true, desc => ?DESC(action_parameters) + } + ) + ); +fields(action_parameters) -> + [ + {sql, + mk( + binary(), + #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>} + )} + ]; +fields("config_connector") -> + emqx_connector_schema:common_fields() ++ + emqx_mysql:fields(config) ++ + emqx_connector_schema:resource_opts_ref(?MODULE, connector_resource_opts); +fields(connector_resource_opts) -> + emqx_connector_schema:resource_opts_fields(); fields("post") -> [type_field(), name_field() | fields("config")]; fields("put") -> fields("config"); fields("get") -> - emqx_bridge_schema:status_fields() ++ fields("post"). + emqx_bridge_schema:status_fields() ++ fields("post"); +fields("get_bridge_v2") -> + emqx_bridge_schema:status_fields() ++ fields("post_bridge_v2"); +fields("post_bridge_v2") -> + [type_field(), name_field() | fields(mysql_action)]; +fields("put_bridge_v2") -> + fields(mysql_action); +fields(Field) when + Field == "get_connector"; + Field == "put_connector"; + Field == "post_connector" +-> + emqx_connector_schema:api_fields( + Field, + ?CONNECTOR_TYPE, + emqx_mysql:fields(config) ++ + emqx_connector_schema:resource_opts_ref(?MODULE, connector_resource_opts) + ). desc("config") -> ?DESC("desc_config"); +desc("config_connector") -> + ?DESC("desc_config"); +desc(connector_resource_opts) -> + ?DESC(emqx_resource_schema, "resource_opts"); +desc(action_parameters) -> + ?DESC(action_parameters); +desc(mysql_action) -> + ?DESC(mysql_action); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for MySQL using `", string:to_upper(Method), "` method."]; desc(_) -> diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_action_info.erl b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_action_info.erl new file mode 100644 index 000000000..31817b02f --- /dev/null +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_action_info.erl @@ -0,0 +1,64 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_mysql_action_info). + +-behaviour(emqx_action_info). + +%% behaviour callbacks +-export([ + action_type_name/0, + bridge_v1_config_to_action_config/2, + bridge_v1_config_to_connector_config/1, + bridge_v1_type_name/0, + connector_action_config_to_bridge_v1_config/2, + connector_type_name/0, + schema_module/0 +]). + +-import(emqx_utils_conv, [bin/1]). + +-define(MYSQL_TYPE, mysql). +-define(SCHEMA_MODULE, emqx_bridge_mysql). + +action_type_name() -> ?MYSQL_TYPE. +bridge_v1_type_name() -> ?MYSQL_TYPE. +connector_type_name() -> ?MYSQL_TYPE. + +schema_module() -> ?SCHEMA_MODULE. + +connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) -> + MergedConfig = + emqx_utils_maps:deep_merge( + maps:without( + [<<"connector">>], + emqx_utils_maps:unindent(<<"parameters">>, ActionConfig) + ), + ConnectorConfig + ), + BridgeV1Keys = schema_keys("config"), + maps:with(BridgeV1Keys, MergedConfig). + +bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) -> + ActionTopLevelKeys = schema_keys(mysql_action), + ActionParametersKeys = schema_keys(action_parameters), + ActionKeys = ActionTopLevelKeys ++ ActionParametersKeys, + ActionConfig = make_config_map(ActionKeys, ActionParametersKeys, BridgeV1Config), + ActionConfig#{<<"connector">> => ConnectorName}. + +bridge_v1_config_to_connector_config(BridgeV1Config) -> + ConnectorKeys = schema_keys("config_connector"), + ResourceOptsKeys = schema_keys(connector_resource_opts), + maps:update_with( + <<"resource_opts">>, + fun(ResourceOpts) -> maps:with(ResourceOptsKeys, ResourceOpts) end, + #{}, + maps:with(ConnectorKeys, BridgeV1Config) + ). + +make_config_map(PickKeys, IndentKeys, Config) -> + Conf0 = maps:with(PickKeys, Config), + emqx_utils_maps:indent(<<"parameters">>, IndentKeys, Conf0). + +schema_keys(Name) -> + [bin(Key) || Key <- proplists:get_keys(?SCHEMA_MODULE:fields(Name))]. diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl new file mode 100644 index 000000000..468f64d1f --- /dev/null +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl @@ -0,0 +1,150 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_mysql_connector). + +-behaviour(emqx_resource). + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% `emqx_resource' API +-export([ + on_remove_channel/3, + callback_mode/0, + on_add_channel/4, + on_batch_query/3, + on_get_channel_status/3, + on_get_channels/1, + on_get_status/2, + on_query/3, + on_start/2, + on_stop/2 +]). + +%%======================================================================================== +%% `emqx_resource' API +%%======================================================================================== + +callback_mode() -> emqx_mysql:callback_mode(). + +on_add_channel( + _InstanceId, + #{channels := Channels, connector_state := ConnectorState} = State0, + ChannelId, + ChannelConfig0 +) -> + ChannelConfig1 = emqx_utils_maps:unindent(parameters, ChannelConfig0), + QueryTemplates = emqx_mysql:parse_prepare_sql(ChannelId, ChannelConfig1), + ChannelConfig2 = maps:merge(ChannelConfig1, QueryTemplates), + ChannelConfig = set_prepares(ChannelConfig2, ConnectorState), + State = State0#{ + channels => maps:put(ChannelId, ChannelConfig, Channels), + connector_state => ConnectorState + }, + {ok, State}. + +on_get_channel_status(_InstanceId, ChannelId, #{channels := Channels}) -> + case maps:get(ChannelId, Channels) of + #{prepares := ok} -> + connected; + #{prepares := {error, _}} -> + connecting + end. + +on_get_channels(InstanceId) -> + emqx_bridge_v2:get_channels_for_connector(InstanceId). + +on_get_status(InstanceId, #{channels := Channels0, connector_state := ConnectorState} = State0) -> + case emqx_mysql:on_get_status(InstanceId, ConnectorState) of + WithState when is_tuple(WithState) -> + NewConnectorState = element(2, WithState), + State = State0#{connector_state => NewConnectorState}, + setelement(2, WithState, State); + connected -> + Channels = + maps:map( + fun + (_ChannelId, #{prepares := ok} = ChannelConfig) -> + ChannelConfig; + (_ChannelId, #{prepares := {error, _}} = ChannelConfig) -> + set_prepares(ChannelConfig, ConnectorState) + end, + Channels0 + ), + State = State0#{channels => Channels}, + {connected, State}; + Other -> + Other + end. + +on_query(InstId, {TypeOrKey, SQLOrKey}, State) -> + on_query(InstId, {TypeOrKey, SQLOrKey, [], default_timeout}, State); +on_query(InstId, {TypeOrKey, SQLOrKey, Params}, State) -> + on_query(InstId, {TypeOrKey, SQLOrKey, Params, default_timeout}, State); +on_query( + InstanceId, + {Channel, _Message, _Params, _Timeout} = Request, + #{channels := Channels, connector_state := ConnectorState} +) when is_binary(Channel) -> + ChannelConfig = maps:get(Channel, Channels), + Result = emqx_mysql:on_query( + InstanceId, + Request, + maps:merge(ConnectorState, ChannelConfig) + ), + ?tp(mysql_connector_on_query_return, #{instance_id => InstanceId, result => Result}), + Result; +on_query(InstanceId, Request, _State = #{channels := _Channels, connector_state := ConnectorState}) -> + emqx_mysql:on_query(InstanceId, Request, ConnectorState). + +on_batch_query( + InstanceId, + [Req | _] = BatchRequest, + #{channels := Channels, connector_state := ConnectorState} +) when is_binary(element(1, Req)) -> + Channel = element(1, Req), + ChannelConfig = maps:get(Channel, Channels), + Result = emqx_mysql:on_batch_query( + InstanceId, + BatchRequest, + maps:merge(ConnectorState, ChannelConfig) + ), + ?tp(mysql_connector_on_batch_query_return, #{instance_id => InstanceId, result => Result}), + Result; +on_batch_query(InstanceId, BatchRequest, _State = #{connector_state := ConnectorState}) -> + emqx_mysql:on_batch_query(InstanceId, BatchRequest, ConnectorState). + +on_remove_channel( + _InstanceId, #{channels := Channels, connector_state := ConnectorState} = State, ChannelId +) -> + ChannelConfig = maps:get(ChannelId, Channels), + emqx_mysql:unprepare_sql(maps:merge(ChannelConfig, ConnectorState)), + NewState = State#{channels => maps:remove(ChannelId, Channels)}, + {ok, NewState}. + +-spec on_start(binary(), hocon:config()) -> + {ok, #{connector_state := emqx_mysql:state(), channels := map()}} | {error, _}. +on_start(InstanceId, Config) -> + case emqx_mysql:on_start(InstanceId, Config) of + {ok, ConnectorState} -> + State = #{ + connector_state => ConnectorState, + channels => #{} + }, + {ok, State}; + {error, Reason} -> + {error, Reason} + end. + +on_stop(InstanceId, _State = #{connector_state := ConnectorState}) -> + ok = emqx_mysql:on_stop(InstanceId, ConnectorState), + ?tp(mysql_connector_stopped, #{instance_id => InstanceId}), + ok. + +%%======================================================================================== +%% Helper fns +%%======================================================================================== +set_prepares(ChannelConfig, ConnectorState) -> + #{prepares := Prepares} = + emqx_mysql:init_prepare(maps:merge(ConnectorState, ChannelConfig)), + ChannelConfig#{prepares => Prepares}. diff --git a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl index 98b957b19..f1b3f8260 100644 --- a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl +++ b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl @@ -242,13 +242,12 @@ send_message(Config, Payload) -> query_resource(Config, Request) -> Name = ?config(mysql_name, Config), BridgeType = ?config(mysql_bridge_type, Config), - ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), - emqx_resource:query(ResourceID, Request, #{timeout => 500}). + emqx_bridge_v2:query(BridgeType, Name, Request, #{timeout => 500}). sync_query_resource(Config, Request) -> Name = ?config(mysql_name, Config), BridgeType = ?config(mysql_bridge_type, Config), - ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + ResourceID = emqx_bridge_v2:id(BridgeType, Name), emqx_resource_buffer_worker:simple_sync_query(ResourceID, Request). query_resource_async(Config, Request) -> @@ -256,8 +255,7 @@ query_resource_async(Config, Request) -> BridgeType = ?config(mysql_bridge_type, Config), Ref = alias([reply]), AsyncReplyFun = fun(Result) -> Ref ! {result, Ref, Result} end, - ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), - Return = emqx_resource:query(ResourceID, Request, #{ + Return = emqx_bridge_v2:query(BridgeType, Name, Request, #{ timeout => 500, async_reply_fun => {AsyncReplyFun, []} }), {Return, Ref}. @@ -274,7 +272,9 @@ unprepare(Config, Key) -> Name = ?config(mysql_name, Config), BridgeType = ?config(mysql_bridge_type, Config), ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), - {ok, _, #{state := #{pool_name := PoolName}}} = emqx_resource:get_instance(ResourceID), + {ok, _, #{state := #{connector_state := #{pool_name := PoolName}}}} = emqx_resource:get_instance( + ResourceID + ), [ begin {ok, Conn} = ecpool_worker:client(Worker), @@ -343,6 +343,17 @@ create_rule_and_action_http(Config) -> Error end. +request_api_status(BridgeId) -> + Path = emqx_mgmt_api_test_util:api_path(["bridges", BridgeId]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(get, Path, "", AuthHeader) of + {ok, Res0} -> + #{<<"status">> := Status} = _Res = emqx_utils_json:decode(Res0, [return_maps]), + {ok, binary_to_existing_atom(Status)}; + Error -> + Error + end. + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -519,14 +530,18 @@ t_write_timeout(Config) -> 2 * Timeout ), emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> + Name = ?config(mysql_name, Config), + BridgeType = ?config(mysql_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + case QueryMode of sync -> ?assertMatch( {error, {resource_error, #{reason := timeout}}}, - query_resource(Config, {send_message, SentData, [], Timeout}) + query_resource(Config, {ResourceID, SentData, [], Timeout}) ); async -> - query_resource(Config, {send_message, SentData, [], Timeout}), + query_resource(Config, {ResourceID, SentData, [], Timeout}), ok end, ok @@ -703,7 +718,10 @@ t_uninitialized_prepared_statement(Config) -> ), Val = integer_to_binary(erlang:unique_integer()), SentData = #{payload => Val, timestamp => 1668602148000}, - unprepare(Config, send_message), + Name = ?config(mysql_name, Config), + BridgeType = ?config(mysql_bridge_type, Config), + ResourceID = emqx_bridge_v2:id(BridgeType, Name), + unprepare(Config, ResourceID), ?check_trace( begin {Res, {ok, _}} = @@ -721,7 +739,7 @@ t_uninitialized_prepared_statement(Config) -> #{?snk_kind := mysql_connector_prepare_query_failed, error := not_prepared}, #{ ?snk_kind := mysql_connector_on_query_prepared_sql, - type_or_key := send_message + type_or_key := ResourceID }, Trace ) @@ -736,33 +754,58 @@ t_uninitialized_prepared_statement(Config) -> ok. t_missing_table(Config) -> + QueryMode = ?config(query_mode, Config), Name = ?config(mysql_name, Config), BridgeType = ?config(mysql_bridge_type, Config), - ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), ?check_trace( begin connect_and_drop_table(Config), ?assertMatch({ok, _}, create_bridge(Config)), + BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name), ?retry( _Sleep = 1_000, _Attempts = 20, ?assertMatch( {ok, Status} when Status == connecting orelse Status == disconnected, - emqx_resource_manager:health_check(ResourceID) + request_api_status(BridgeID) ) ), Val = integer_to_binary(erlang:unique_integer()), SentData = #{payload => Val, timestamp => 1668602148000}, - Timeout = 1000, - ?assertMatch( - {error, {resource_error, #{reason := unhealthy_target}}}, - query_resource(Config, {send_message, SentData, [], Timeout}) - ), + %Timeout = 1000, + ResourceID = emqx_bridge_v2:id(BridgeType, Name), + Request = {ResourceID, SentData}, + Result = + case QueryMode of + sync -> + query_resource(Config, Request); + async -> + {_, Ref} = query_resource_async(Config, Request), + {ok, Res} = receive_result(Ref, 2_000), + Res + end, + + BatchSize = ?config(batch_size, Config), + IsBatch = BatchSize > 1, + case IsBatch of + true -> + ?assertMatch( + {error, + {unrecoverable_error, + {1146, <<"42S02">>, <<"Table 'mqtt.mqtt_test' doesn't exist">>}}}, + Result + ); + false -> + ?assertMatch( + {error, undefined_table}, + Result + ) + end, ok end, fun(Trace) -> - ?assertMatch([_, _, _], ?of_kind(mysql_undefined_table, Trace)), + ?assertMatch([_ | _], ?of_kind(mysql_undefined_table, Trace)), ok end ). @@ -770,9 +813,9 @@ t_missing_table(Config) -> t_table_removed(Config) -> Name = ?config(mysql_name, Config), BridgeType = ?config(mysql_bridge_type, Config), - ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), connect_and_create_table(Config), ?assertMatch({ok, _}, create_bridge(Config)), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), ?retry( _Sleep = 1_000, _Attempts = 20, @@ -782,17 +825,17 @@ t_table_removed(Config) -> Val = integer_to_binary(erlang:unique_integer()), SentData = #{payload => Val, timestamp => 1668602148000}, Timeout = 1000, + ActionID = emqx_bridge_v2:id(BridgeType, Name), ?assertMatch( {error, {unrecoverable_error, {1146, <<"42S02">>, <<"Table 'mqtt.mqtt_test' doesn't exist">>}}}, - sync_query_resource(Config, {send_message, SentData, [], Timeout}) + sync_query_resource(Config, {ActionID, SentData, [], Timeout}) ), ok. t_nested_payload_template(Config) -> Name = ?config(mysql_name, Config), BridgeType = ?config(mysql_bridge_type, Config), - ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), Value = integer_to_binary(erlang:unique_integer()), {ok, _} = create_bridge( Config, @@ -803,6 +846,7 @@ t_nested_payload_template(Config) -> } ), {ok, #{<<"from">> := [Topic]}} = create_rule_and_action_http(Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), ?retry( _Sleep = 1_000, _Attempts = 20, diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 1ca6e4a5d..4e8618915 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -34,6 +34,8 @@ resource_type(matrix) -> emqx_postgresql; resource_type(mongodb) -> emqx_bridge_mongodb_connector; +resource_type(mysql) -> + emqx_bridge_mysql_connector; resource_type(pgsql) -> emqx_postgresql; resource_type(syskeeper_forwarder) -> @@ -94,6 +96,14 @@ connector_structs() -> required => false } )}, + {matrix, + mk( + hoconsc:map(name, ref(emqx_bridge_matrix, "config_connector")), + #{ + desc => <<"Matrix Connector Config">>, + required => false + } + )}, {mongodb, mk( hoconsc:map(name, ref(emqx_bridge_mongodb, "config_connector")), @@ -102,6 +112,30 @@ connector_structs() -> required => false } )}, + {mysql, + mk( + hoconsc:map(name, ref(emqx_bridge_mysql, "config_connector")), + #{ + desc => <<"MySQL Connector Config">>, + required => false + } + )}, + {pgsql, + mk( + hoconsc:map(name, ref(emqx_bridge_pgsql, "config_connector")), + #{ + desc => <<"PostgreSQL Connector Config">>, + required => false + } + )}, + {redis, + mk( + hoconsc:map(name, ref(emqx_bridge_redis_schema, "config_connector")), + #{ + desc => <<"Redis Connector Config">>, + required => false + } + )}, {syskeeper_forwarder, mk( hoconsc:map(name, ref(emqx_bridge_syskeeper_connector, config)), @@ -118,14 +152,6 @@ connector_structs() -> required => false } )}, - {pgsql, - mk( - hoconsc:map(name, ref(emqx_bridge_pgsql, "config_connector")), - #{ - desc => <<"PostgreSQL Connector Config">>, - required => false - } - )}, {timescale, mk( hoconsc:map(name, ref(emqx_bridge_timescale, "config_connector")), @@ -133,22 +159,6 @@ connector_structs() -> desc => <<"Timescale Connector Config">>, required => false } - )}, - {matrix, - mk( - hoconsc:map(name, ref(emqx_bridge_matrix, "config_connector")), - #{ - desc => <<"Matrix Connector Config">>, - required => false - } - )}, - {redis, - mk( - hoconsc:map(name, ref(emqx_bridge_redis_schema, "config_connector")), - #{ - desc => <<"Redis Connector Config">>, - required => false - } )} ]. @@ -160,6 +170,7 @@ schema_modules() -> emqx_bridge_kafka, emqx_bridge_matrix, emqx_bridge_mongodb, + emqx_bridge_mysql, emqx_bridge_syskeeper_connector, emqx_bridge_syskeeper_proxy, emqx_bridge_timescale, @@ -185,6 +196,7 @@ api_schemas(Method) -> api_ref(emqx_bridge_kafka, <<"kafka_producer">>, Method ++ "_connector"), api_ref(emqx_bridge_matrix, <<"matrix">>, Method ++ "_connector"), api_ref(emqx_bridge_mongodb, <<"mongodb">>, Method ++ "_connector"), + api_ref(emqx_bridge_mysql, <<"mysql">>, Method ++ "_connector"), api_ref(emqx_bridge_syskeeper_connector, <<"syskeeper_forwarder">>, Method), api_ref(emqx_bridge_syskeeper_proxy, <<"syskeeper_proxy">>, Method), api_ref(emqx_bridge_timescale, <<"timescale">>, Method ++ "_connector"), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 39ae3e764..6515a45d8 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -36,9 +36,11 @@ -export([get_response/0, put_request/0, post_request/0]). -export([connector_type_to_bridge_types/1]). + -export([ api_fields/3, common_fields/0, + connector_values/3, status_and_actions_fields/0, type_and_name_fields/1 ]). @@ -128,16 +130,18 @@ connector_type_to_bridge_types(matrix) -> [matrix]; connector_type_to_bridge_types(mongodb) -> [mongodb, mongodb_rs, mongodb_sharded, mongodb_single]; +connector_type_to_bridge_types(mysql) -> + [mysql]; connector_type_to_bridge_types(pgsql) -> [pgsql]; +connector_type_to_bridge_types(redis) -> + [redis, redis_single, redis_sentinel, redis_cluster]; connector_type_to_bridge_types(syskeeper_forwarder) -> [syskeeper_forwarder]; connector_type_to_bridge_types(syskeeper_proxy) -> []; connector_type_to_bridge_types(timescale) -> - [timescale]; -connector_type_to_bridge_types(redis) -> - [redis, redis_single, redis_sentinel, redis_cluster]. + [timescale]. actions_config_name() -> <<"actions">>. @@ -549,6 +553,48 @@ resource_opts_fields(Overrides) -> emqx_resource_schema:create_opts(Overrides) ). +-type http_method() :: get | post | put. +-type schema_example_map() :: #{atom() => term()}. + +-spec connector_values(http_method(), atom(), schema_example_map()) -> schema_example_map(). +connector_values(Method, Type, ConnectorValues) -> + TypeBin = atom_to_binary(Type), + lists:foldl( + fun(M1, M2) -> + maps:merge(M1, M2) + end, + #{ + description => <<"My example ", TypeBin/binary, " connector">> + }, + [ + ConnectorValues, + method_values(Method, Type) + ] + ). + +method_values(post, Type) -> + TypeBin = atom_to_binary(Type), + #{ + name => <>, + type => TypeBin + }; +method_values(get, Type) -> + maps:merge( + method_values(post, Type), + #{ + status => <<"connected">>, + node_status => [ + #{ + node => <<"emqx@localhost">>, + status => <<"connected">> + } + ], + actions => [<<"my_action">>] + } + ); +method_values(put, _Type) -> + #{}. + %%====================================================================================== %% Helper Functions %%====================================================================================== diff --git a/apps/emqx_mysql/rebar.config b/apps/emqx_mysql/rebar.config index fc7f4df7a..657daf61b 100644 --- a/apps/emqx_mysql/rebar.config +++ b/apps/emqx_mysql/rebar.config @@ -3,7 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ %% NOTE: mind ecpool version when updating eredis_cluster version - {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.4"}}}, + {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.4.1"}}}, {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}} ]}. diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index 66fce9fde..fcfabd61e 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -36,7 +36,13 @@ %% ecpool connect & reconnect -export([connect/1, prepare_sql_to_conn/2]). --export([prepare_sql/2]). +-export([ + init_prepare/1, + prepare_sql/2, + parse_prepare_sql/1, + parse_prepare_sql/2, + unprepare_sql/1 +]). -export([roots/0, fields/1]). @@ -51,9 +57,10 @@ #{ pool_name := binary(), prepares := ok | {error, _}, - templates := #{{atom(), batch | prepstmt} => template()} + templates := #{{atom(), batch | prepstmt} => template()}, + query_templates := map() }. - +-export_type([state/0]). %%===================================================================== %% Hocon schema roots() -> @@ -62,8 +69,7 @@ roots() -> fields(config) -> [{server, server()}] ++ add_default_username(emqx_connector_schema_lib:relational_db_fields(), []) ++ - emqx_connector_schema_lib:ssl_fields() ++ - emqx_connector_schema_lib:prepare_statement_fields(). + emqx_connector_schema_lib:ssl_fields(). add_default_username([{username, OrigUsernameFn} | Tail], Head) -> Head ++ [{username, add_default_fn(OrigUsernameFn, <<"root">>)} | Tail]; @@ -267,7 +273,7 @@ do_check_prepares( ); do_check_prepares(#{prepares := ok}) -> ok; -do_check_prepares(#{prepares := {error, _}} = State) -> +do_check_prepares(#{prepares := {error, _}, query_templates := _} = State) -> %% retry to prepare case prepare_sql(State) of ok -> @@ -275,7 +281,9 @@ do_check_prepares(#{prepares := {error, _}} = State) -> {ok, State#{prepares => ok}}; {error, Reason} -> {error, Reason} - end. + end; +do_check_prepares(_NoTemplates) -> + ok. %% =================================================================== @@ -323,16 +331,18 @@ prepare_sql(Templates, PoolName) -> end. do_prepare_sql(Templates, PoolName) -> - Conns = - [ - begin - {ok, Conn} = ecpool_worker:client(Worker), - Conn - end - || {_Name, Worker} <- ecpool:workers(PoolName) - ], + Conns = get_connections_from_pool(PoolName), prepare_sql_to_conn_list(Conns, Templates). +get_connections_from_pool(PoolName) -> + [ + begin + {ok, Conn} = ecpool_worker:client(Worker), + Conn + end + || {_Name, Worker} <- ecpool:workers(PoolName) + ]. + prepare_sql_to_conn_list([], _Templates) -> ok; prepare_sql_to_conn_list([Conn | ConnList], Templates) -> @@ -369,6 +379,18 @@ prepare_sql_to_conn(Conn, [{{Key, prepstmt}, {SQL, _RowTemplate}} | Rest]) -> prepare_sql_to_conn(Conn, [{_Key, _Template} | Rest]) -> prepare_sql_to_conn(Conn, Rest). +unprepare_sql(#{query_templates := Templates, pool_name := PoolName}) -> + ecpool:remove_reconnect_callback(PoolName, {?MODULE, prepare_sql_to_conn}), + lists:foreach( + fun(Conn) -> + lists:foreach( + fun(Template) -> unprepare_sql_to_conn(Conn, Template) end, + maps:to_list(Templates) + ) + end, + get_connections_from_pool(PoolName) + ). + unprepare_sql_to_conn(Conn, {{Key, prepstmt}, _}) -> mysql:unprepare(Conn, Key); unprepare_sql_to_conn(Conn, Key) when is_atom(Key) -> @@ -377,12 +399,15 @@ unprepare_sql_to_conn(_Conn, _) -> ok. parse_prepare_sql(Config) -> + parse_prepare_sql(send_message, Config). + +parse_prepare_sql(Key, Config) -> Queries = case Config of #{prepare_statement := Qs} -> Qs; #{sql := Query} -> - #{send_message => Query}; + #{Key => Query}; _ -> #{} end, @@ -436,7 +461,9 @@ proc_sql_params(TypeOrKey, SQLOrData, Params, #{query_templates := Templates}) - {emqx_jsonish, SQLOrData} ), {TypeOrKey, Row} - end. + end; +proc_sql_params(_TypeOrKey, SQLOrData, Params, _State) -> + {SQLOrData, Params}. on_batch_insert(InstId, BatchReqs, {InsertPart, RowTemplate}, State) -> Rows = [render_row(RowTemplate, Msg) || {_, Msg} <- BatchReqs], diff --git a/apps/emqx_utils/src/emqx_utils_maps.erl b/apps/emqx_utils/src/emqx_utils_maps.erl index 043ab5210..1d97a926a 100644 --- a/apps/emqx_utils/src/emqx_utils_maps.erl +++ b/apps/emqx_utils/src/emqx_utils_maps.erl @@ -16,27 +16,29 @@ -module(emqx_utils_maps). -export([ - deep_get/2, - deep_get/3, - deep_find/2, - deep_put/3, - deep_force_put/3, - deep_remove/2, - deep_merge/2, + best_effort_recursive_sum/3, binary_key_map/1, - safe_atom_key_map/1, - unsafe_atom_key_map/1, - jsonable_map/1, - jsonable_map/2, binary_string/1, deep_convert/3, + deep_find/2, + deep_force_put/3, + deep_get/2, + deep_get/3, + deep_merge/2, + deep_put/3, + deep_remove/2, diff_maps/2, - best_effort_recursive_sum/3, if_only_to_toggle_enable/2, - update_if_present/3, + indent/3, + jsonable_map/1, + jsonable_map/2, + key_comparer/1, put_if/4, rename/3, - key_comparer/1 + safe_atom_key_map/1, + unindent/2, + unsafe_atom_key_map/1, + update_if_present/3 ]). -export_type([config_key/0, config_key_path/0]). @@ -332,3 +334,18 @@ key_comparer(K) -> (M1, M2) -> M1 < M2 end. + +-spec indent(term(), [term()], map()) -> map(). +indent(IndentKey, PickKeys, Map) -> + maps:put( + IndentKey, + maps:with(PickKeys, Map), + maps:without(PickKeys, Map) + ). + +-spec unindent(term(), map()) -> map(). +unindent(Key, Map) -> + maps:merge( + maps:remove(Key, Map), + maps:get(Key, Map, #{}) + ). diff --git a/apps/emqx_utils/test/emqx_utils_maps_tests.erl b/apps/emqx_utils/test/emqx_utils_maps_tests.erl index a9f39536e..2778b5257 100644 --- a/apps/emqx_utils/test/emqx_utils_maps_tests.erl +++ b/apps/emqx_utils/test/emqx_utils_maps_tests.erl @@ -17,6 +17,8 @@ -module(emqx_utils_maps_tests). -include_lib("eunit/include/eunit.hrl"). +-import(emqx_utils_maps, [indent/3, unindent/2]). + best_effort_recursive_sum_test_() -> DummyLogger = fun(_) -> ok end, [ @@ -129,3 +131,44 @@ key_comparer_test() -> #{} ]) ). + +map_indent_unindent_test_() -> + M = #{a => 1, b => 2}, + [ + ?_assertEqual( + #{a => 1, c => #{b => 2}}, + indent(c, [b], M) + ), + ?_assertEqual( + M, + unindent(c, indent(c, [b], M)) + ), + ?_assertEqual( + #{a => 1, b => #{b => 2}}, + indent(b, [b], M) + ), + ?_assertEqual( + M, + unindent(b, #{a => 1, b => #{b => 2}}) + ), + ?_assertEqual( + #{a => 2}, + unindent(b, #{a => 1, b => #{a => 2}}) + ), + ?_assertEqual( + #{c => #{a => 1, b => 2}}, + indent(c, [a, b], M) + ), + ?_assertEqual( + #{a => 1, b => 2, c => #{}}, + indent(c, [], M) + ), + ?_assertEqual( + #{a => 1, b => 2, c => #{}}, + indent(c, [d, e, f], M) + ), + ?_assertEqual( + #{a => 1, b => 2}, + unindent(c, M) + ) + ]. diff --git a/mix.exs b/mix.exs index 21a07dee4..74f4ae105 100644 --- a/mix.exs +++ b/mix.exs @@ -59,7 +59,7 @@ defmodule EMQXUmbrella.MixProject do {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.0", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.15", override: true}, - {:ecpool, github: "emqx/ecpool", tag: "0.5.4", override: true}, + {:ecpool, github: "emqx/ecpool", tag: "0.5.7", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, # maybe forbid to fetch quicer diff --git a/rebar.config b/rebar.config index 830ffc051..7380347f2 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.0"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.12"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.15"}}} - , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.4"}}} + , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.7"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.10.1"}}} diff --git a/rel/i18n/emqx_bridge_mysql.hocon b/rel/i18n/emqx_bridge_mysql.hocon index 37326be81..057b1b145 100644 --- a/rel/i18n/emqx_bridge_mysql.hocon +++ b/rel/i18n/emqx_bridge_mysql.hocon @@ -40,4 +40,14 @@ sql_template.desc: sql_template.label: """SQL Template""" +action_parameters.label: +"""Action Parameters""" +action_parameters.desc: +"""Additional parameters specific to this action type""" + +mysql_action.label: +"""MySQL Action""" +mysql_action.desc: +"""Action to interact with a MySQL connector""" + } From b5de58df2dcf8f01eaa339cf7ba8738c5d77693d Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Fri, 8 Dec 2023 18:44:19 +0200 Subject: [PATCH 10/50] chore: upgrade opentelemetry lib to 1.4.7-emqx --- apps/emqx_opentelemetry/rebar.config | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_opentelemetry/rebar.config b/apps/emqx_opentelemetry/rebar.config index a14c5922e..5addc4a7e 100644 --- a/apps/emqx_opentelemetry/rebar.config +++ b/apps/emqx_opentelemetry/rebar.config @@ -3,13 +3,13 @@ {deps, [{emqx, {path, "../emqx"}} %% trace - , {opentelemetry_api, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.6-emqx"}, "apps/opentelemetry_api"}} - , {opentelemetry, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.6-emqx"}, "apps/opentelemetry"}} + , {opentelemetry_api, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.7-emqx"}, "apps/opentelemetry_api"}} + , {opentelemetry, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.7-emqx"}, "apps/opentelemetry"}} %% logs, metrics - , {opentelemetry_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.6-emqx"}, "apps/opentelemetry_experimental"}} - , {opentelemetry_api_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.6-emqx"}, "apps/opentelemetry_api_experimental"}} + , {opentelemetry_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.7-emqx"}, "apps/opentelemetry_experimental"}} + , {opentelemetry_api_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.7-emqx"}, "apps/opentelemetry_api_experimental"}} %% export - , {opentelemetry_exporter, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.6-emqx"}, "apps/opentelemetry_exporter"}} + , {opentelemetry_exporter, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.7-emqx"}, "apps/opentelemetry_exporter"}} ]}. {edoc_opts, [{preprocess, true}]}. From ddbb8560fa3043315e44721658d3432fdbde6abd Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 8 Dec 2023 17:58:53 +0100 Subject: [PATCH 11/50] fix(dialyzer): batch 2 --- .../emqx_authn_password_hashing.erl | 5 ++-- .../emqx_authn/proto/emqx_authn_proto_v1.erl | 2 +- .../src/emqx_authz/emqx_authz_rule.erl | 29 +++++++++++++++---- .../emqx_authz/proto/emqx_authz_proto_v1.erl | 2 +- .../src/emqx_auth_mnesia.app.src | 5 ++-- .../src/emqx_authz_mnesia.erl | 6 ++-- .../src/proto/emqx_bridge_proto_v1.erl | 6 ++-- .../src/proto/emqx_bridge_proto_v2.erl | 8 ++--- .../src/proto/emqx_bridge_proto_v3.erl | 8 ++--- .../src/proto/emqx_bridge_proto_v4.erl | 8 ++--- .../src/proto/emqx_bridge_proto_v5.erl | 14 ++++----- .../src/emqx_bridge_mqtt_egress.erl | 3 +- .../src/emqx_bridge_pulsar_impl_producer.erl | 6 ++-- .../emqx_bridge_syskeeper_proxy_server.erl | 2 +- apps/emqx_conf/src/emqx_conf.erl | 4 +-- .../src/emqx_connector_jwt_sup.erl | 2 +- .../src/proto/emqx_connector_proto_v1.erl | 4 +-- .../src/emqx_dashboard_swagger.erl | 4 +-- apps/emqx_durable_storage/src/emqx_ds.erl | 1 + .../emqx_durable_storage/src/emqx_ds_conf.erl | 20 ++++++++----- apps/emqx_durable_storage/src/emqx_ds_lts.erl | 7 ++++- .../src/emqx_ds_replication_layer.erl | 6 ++-- .../src/emqx_ds_replication_layer_meta.erl | 2 +- .../src/emqx_ds_storage_layer.erl | 27 ++++++++++++----- .../src/emqx_ds_storage_layer_sup.erl | 2 +- .../src/proto/emqx_ds_proto_v1.erl | 5 ++-- apps/emqx_exhook/src/emqx_exhook_mgr.erl | 4 +++ apps/emqx_exhook/src/emqx_exhook_sup.erl | 6 +++- apps/emqx_ft/src/emqx_ft.app.src | 2 +- apps/emqx_ft/src/emqx_ft_api.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 3 +- .../emqx_ft/src/emqx_ft_storage_fs_reader.erl | 4 +-- .../src/bhvrs/emqx_gateway_channel.erl | 8 ++--- .../src/bhvrs/emqx_gateway_conn.erl | 2 +- .../src/bhvrs/emqx_gateway_frame.erl | 14 ++++----- .../src/emqx_gateway_registry.erl | 2 ++ apps/emqx_utils/src/emqx_placeholder.erl | 2 ++ 38 files changed, 148 insertions(+), 91 deletions(-) diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl index 16af4fd23..0f5aee691 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl @@ -201,7 +201,8 @@ gen_salt(#{name := Other}) when Other =/= plain, Other =/= bcrypt -> <> = crypto:strong_rand_bytes(16), iolist_to_binary(io_lib:format("~32.16.0b", [X])). --spec hash(algorithm_rw(), emqx_passwd:password()) -> {emqx_passwd:hash(), emqx_passwd:salt()}. +-spec hash(algorithm_rw(), emqx_passwd:password()) -> + {emqx_passwd:password_hash(), emqx_passwd:salt()}. hash(#{name := bcrypt, salt_rounds := _} = Algorithm, Password) -> Salt0 = gen_salt(Algorithm), Hash = emqx_passwd:hash({bcrypt, Salt0}, Password), @@ -231,7 +232,7 @@ hash(#{name := Other, salt_position := SaltPosition} = Algorithm, Password) -> -spec check_password( algorithm(), emqx_passwd:salt(), - emqx_passwd:hash(), + emqx_passwd:password_hash(), emqx_passwd:password() ) -> boolean(). check_password(#{name := bcrypt}, _Salt, PasswordHash, Password) -> diff --git a/apps/emqx_auth/src/emqx_authn/proto/emqx_authn_proto_v1.erl b/apps/emqx_auth/src/emqx_authn/proto/emqx_authn_proto_v1.erl index cb0f2641c..3272ddcf7 100644 --- a/apps/emqx_auth/src/emqx_authn/proto/emqx_authn_proto_v1.erl +++ b/apps/emqx_auth/src/emqx_authn/proto/emqx_authn_proto_v1.erl @@ -31,7 +31,7 @@ introduced_in() -> "5.0.0". -spec lookup_from_all_nodes([node()], atom(), binary()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(term()). lookup_from_all_nodes(Nodes, ChainName, AuthenticatorID) -> erpc:multicall( Nodes, emqx_authn_api, lookup_from_local_node, [ChainName, AuthenticatorID], ?TIMEOUT diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl index ad6dec56b..eba0ee554 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl @@ -16,10 +16,6 @@ -module(emqx_authz_rule). --include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_placeholder.hrl"). --include("emqx_authz.hrl"). - -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). @@ -29,9 +25,16 @@ -export([ match/4, matches/4, - compile/1 + compile/1, + compile/4 ]). +-export_type([action/0, action_precompile/0]). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_placeholder.hrl"). +-include("emqx_authz.hrl"). + -type permission() :: allow | deny. -type who_condition() :: @@ -73,8 +76,24 @@ topic_condition/0 ]). +-type action_precompile() :: + subscribe + | publish + | {subscribe, list()} + | {publish, list()} + | all. + +-type topic_filter() :: emqx_types:topic(). + +-type rule_precompile() :: {permission(), who_condition(), action_precompile(), [topic_filter()]}. + -define(IS_PERMISSION(Permission), (Permission =:= allow orelse Permission =:= deny)). +-spec compile(permission(), who_condition(), action_precompile(), [topic_filter()]) -> rule(). +compile(Permission, Who, Action, TopicFilters) -> + compile({Permission, Who, Action, TopicFilters}). + +-spec compile({permission(), all} | rule_precompile()) -> rule(). compile({Permission, all}) when ?IS_PERMISSION(Permission) -> diff --git a/apps/emqx_auth/src/emqx_authz/proto/emqx_authz_proto_v1.erl b/apps/emqx_auth/src/emqx_authz/proto/emqx_authz_proto_v1.erl index 1671b39a3..d8d949ba2 100644 --- a/apps/emqx_auth/src/emqx_authz/proto/emqx_authz_proto_v1.erl +++ b/apps/emqx_auth/src/emqx_authz/proto/emqx_authz_proto_v1.erl @@ -31,6 +31,6 @@ introduced_in() -> "5.0.0". -spec lookup_from_all_nodes([node()], atom()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(term()). lookup_from_all_nodes(Nodes, Type) -> erpc:multicall(Nodes, emqx_authz_api_sources, lookup_from_local_node, [Type], ?TIMEOUT). diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src index 5cc2c2a31..3dbdf6625 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src @@ -1,14 +1,15 @@ %% -*- mode: erlang -*- {application, emqx_auth_mnesia, [ {description, "EMQX Buitl-in Database Authentication and Authorization"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {mod, {emqx_auth_mnesia_app, []}}, {applications, [ kernel, stdlib, emqx, - emqx_auth + emqx_auth, + esasl ]}, {env, []}, {modules, []}, diff --git a/apps/emqx_auth_mnesia/src/emqx_authz_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_authz_mnesia.erl index 7e8e463b3..27000b7a3 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authz_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authz_mnesia.erl @@ -32,7 +32,9 @@ -type clientid() :: {clientid, binary()}. -type who() :: username() | clientid() | all. --type rule() :: {emqx_authz_rule:permission(), emqx_authz_rule:action(), emqx_types:topic()}. +-type rule() :: { + emqx_authz_rule:permission(), emqx_authz_rule:action_precompile(), emqx_types:topic() +}. -type rules() :: [rule()]. -record(emqx_acl, { @@ -223,7 +225,7 @@ do_get_rules(Key) -> do_authorize(_Client, _PubSub, _Topic, []) -> nomatch; do_authorize(Client, PubSub, Topic, [{Permission, Action, TopicFilter} | Tail]) -> - Rule = emqx_authz_rule:compile({Permission, all, Action, [TopicFilter]}), + Rule = emqx_authz_rule:compile(Permission, all, Action, [TopicFilter]), case emqx_authz_rule:match(Client, PubSub, Topic, Rule) of {matched, Permission} -> {matched, Permission}; nomatch -> do_authorize(Client, PubSub, Topic, Tail) diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v1.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v1.erl index 88554893b..2d65d4506 100644 --- a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v1.erl +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v1.erl @@ -69,7 +69,7 @@ stop_bridge_to_node(Node, BridgeType, BridgeName) -> ). -spec restart_bridges_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -80,7 +80,7 @@ restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec stop_bridges_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -91,7 +91,7 @@ stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec lookup_from_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(term()). lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v2.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v2.erl index bcf6ca198..bd58f6d35 100644 --- a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v2.erl +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v2.erl @@ -82,7 +82,7 @@ stop_bridge_to_node(Node, BridgeType, BridgeName) -> ). -spec restart_bridges_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -93,7 +93,7 @@ restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec start_bridges_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). start_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -104,7 +104,7 @@ start_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec stop_bridges_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -115,7 +115,7 @@ stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec lookup_from_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(term()). lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl index 0b496364a..2724318aa 100644 --- a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl @@ -88,7 +88,7 @@ stop_bridge_to_node(Node, BridgeType, BridgeName) -> ). -spec restart_bridges_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -99,7 +99,7 @@ restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec start_bridges_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). start_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -110,7 +110,7 @@ start_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec stop_bridges_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -121,7 +121,7 @@ stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec lookup_from_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(term()). lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v4.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v4.erl index 937065e41..d90c661f1 100644 --- a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v4.erl +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v4.erl @@ -80,7 +80,7 @@ stop_bridge_to_node(Node, BridgeType, BridgeName) -> ). -spec restart_bridges_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -91,7 +91,7 @@ restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec start_bridges_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). start_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -102,7 +102,7 @@ start_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec stop_bridges_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -113,7 +113,7 @@ stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec lookup_from_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(term()). lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v5.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v5.erl index 75b99f0ec..5995b96f1 100644 --- a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v5.erl +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v5.erl @@ -86,7 +86,7 @@ stop_bridge_to_node(Node, BridgeType, BridgeName) -> ). -spec restart_bridges_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -97,7 +97,7 @@ restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec start_bridges_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). start_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -108,7 +108,7 @@ start_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec stop_bridges_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -119,7 +119,7 @@ stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec lookup_from_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(term()). lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -147,7 +147,7 @@ v2_list_bridges_on_nodes(Nodes) -> erpc:multicall(Nodes, emqx_bridge_v2, list, [], ?TIMEOUT). -spec v2_lookup_from_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(term()). v2_lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, @@ -158,7 +158,7 @@ v2_lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> ). -spec v2_get_metrics_from_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(term()). v2_get_metrics_from_all_nodes(Nodes, ActionType, ActionName) -> erpc:multicall( Nodes, @@ -169,7 +169,7 @@ v2_get_metrics_from_all_nodes(Nodes, ActionType, ActionName) -> ). -spec v2_start_bridge_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(ok). v2_start_bridge_to_all_nodes(Nodes, BridgeType, BridgeName) -> erpc:multicall( Nodes, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl index 2d50d92ce..a9415294c 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl @@ -104,8 +104,7 @@ connect(Pid, Name) -> config(#{remote := RC = #{}} = Conf) -> Conf#{remote => emqx_bridge_mqtt_msg:parse(RC)}. --spec send(pid(), message(), egress()) -> - ok. +-spec send(pid(), message(), egress()) -> ok. send(Pid, MsgIn, Egress) -> emqtt:publish(Pid, export_msg(MsgIn, Egress)). diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl index fed0142c5..53847ff5f 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl @@ -23,7 +23,7 @@ -type state() :: #{ pulsar_client_id := pulsar_client_id(), producers := pulsar_producers:producers(), - sync_timeout := infinity | time:time(), + sync_timeout := erlang:timeout(), message_template := message_template() }. -type buffer_mode() :: memory | disk | hybrid. @@ -43,8 +43,8 @@ bridge_name := atom(), buffer := #{ mode := buffer_mode(), - per_partition_limit := emqx_schema:byte_size(), - segment_bytes := emqx_schema:byte_size(), + per_partition_limit := emqx_schema:bytesize(), + segment_bytes := emqx_schema:bytesize(), memory_overload_protection := boolean() }, compression := compression_mode(), diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl index 057d7579c..187ca1c64 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl @@ -31,7 +31,7 @@ socket := inet:socket(), frame_state := undefined - | emqx_bridge_sysk_frame:state(), + | emqx_bridge_syskeeper_frame:state(), buffer := binary(), conf := map() }. diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 0d2ee72e4..e452bc58e 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -43,12 +43,12 @@ %% API %% @doc Adds a new config handler to emqx_config_handler. --spec add_handler(emqx_config:config_key_path(), module()) -> ok. +-spec add_handler(emqx_utils_maps:config_key_path(), module()) -> ok. add_handler(ConfKeyPath, HandlerName) -> emqx_config_handler:add_handler(ConfKeyPath, HandlerName). %% @doc remove config handler from emqx_config_handler. --spec remove_handler(emqx_config:config_key_path()) -> ok. +-spec remove_handler(emqx_utils_maps:config_key_path()) -> ok. remove_handler(ConfKeyPath) -> emqx_config_handler:remove_handler(ConfKeyPath). diff --git a/apps/emqx_connector/src/emqx_connector_jwt_sup.erl b/apps/emqx_connector/src/emqx_connector_jwt_sup.erl index 4076cc725..764352a8a 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt_sup.erl +++ b/apps/emqx_connector/src/emqx_connector_jwt_sup.erl @@ -48,7 +48,7 @@ init([]) -> %% `emqx_connector_jwt_sup:ensure_jwt/1' to ensure that a JWT has %% been stored, if synchronization is needed. -spec ensure_worker_present(worker_id(), map()) -> - {ok, supervisor:child()} | {error, term()}. + {ok, pid()} | {error, term()}. ensure_worker_present(Id, Config) -> ChildSpec = jwt_worker_child_spec(Id, Config), case supervisor:start_child(?MODULE, ChildSpec) of diff --git a/apps/emqx_connector/src/proto/emqx_connector_proto_v1.erl b/apps/emqx_connector/src/proto/emqx_connector_proto_v1.erl index 0cfb831e8..74a6349e5 100644 --- a/apps/emqx_connector/src/proto/emqx_connector_proto_v1.erl +++ b/apps/emqx_connector/src/proto/emqx_connector_proto_v1.erl @@ -42,7 +42,7 @@ list_connectors_on_nodes(Nodes) -> -type key() :: atom() | binary() | [byte()]. -spec lookup_from_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(term()). lookup_from_all_nodes(Nodes, ConnectorType, ConnectorName) -> erpc:multicall( Nodes, @@ -64,7 +64,7 @@ start_connector_to_node(Node, ConnectorType, ConnectorName) -> ). -spec start_connectors_to_all_nodes([node()], key(), key()) -> - emqx_rpc:erpc_multicall(). + emqx_rpc:erpc_multicall(term()). start_connectors_to_all_nodes(Nodes, ConnectorType, ConnectorName) -> erpc:multicall( Nodes, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 022c6fcb0..a62c205ea 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -181,12 +181,12 @@ fields(hasnext) -> fields(meta) -> fields(page) ++ fields(limit) ++ fields(count) ++ fields(hasnext). --spec schema_with_example(hocon_schema:type(), term()) -> hocon_schema:field_schema_map(). +-spec schema_with_example(hocon_schema:type(), term()) -> hocon_schema:field_schema(). schema_with_example(Type, Example) -> hoconsc:mk(Type, #{examples => #{<<"example">> => Example}}). -spec schema_with_examples(hocon_schema:type(), map() | list(tuple())) -> - hocon_schema:field_schema_map(). + hocon_schema:field_schema(). schema_with_examples(Type, Examples) -> hoconsc:mk(Type, #{examples => #{<<"examples">> => Examples}}). diff --git a/apps/emqx_durable_storage/src/emqx_ds.erl b/apps/emqx_durable_storage/src/emqx_ds.erl index 649341eb5..4516a3c86 100644 --- a/apps/emqx_durable_storage/src/emqx_ds.erl +++ b/apps/emqx_durable_storage/src/emqx_ds.erl @@ -43,6 +43,7 @@ stream_rank/0, iterator/0, message_id/0, + message_store_opts/0, next_result/1, next_result/0, store_batch_result/0, make_iterator_result/1, make_iterator_result/0, diff --git a/apps/emqx_durable_storage/src/emqx_ds_conf.erl b/apps/emqx_durable_storage/src/emqx_ds_conf.erl index 5633cdf58..d9e1efd57 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_conf.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_conf.erl @@ -11,11 +11,17 @@ -export([iteration_options/1]). -export([default_iteration_options/0]). +-export_type([ + backend_config/0, + iteration_options/0 +]). + -type backend_config() :: - {emqx_ds_message_storage_bitmask, emqx_ds_message_storage_bitmask:options()} + {emqx_ds_message_storage_bitmask, emqx_ds_storage_bitfield_lts:options()} | {module(), _Options}. --export_type([backend_config/0]). +-type keyspace() :: atom(). +-type iteration_options() :: map(). %%================================================================================ %% API funcions @@ -23,7 +29,7 @@ -define(APP, emqx_ds). --spec keyspace_config(emqx_ds:keyspace()) -> backend_config(). +-spec keyspace_config(keyspace()) -> backend_config(). keyspace_config(Keyspace) -> DefaultKeyspaceConfig = application:get_env( ?APP, @@ -33,8 +39,8 @@ keyspace_config(Keyspace) -> Keyspaces = application:get_env(?APP, keyspace_config, #{}), maps:get(Keyspace, Keyspaces, DefaultKeyspaceConfig). --spec iteration_options(emqx_ds:keyspace()) -> - emqx_ds_message_storage_bitmask:iteration_options(). +-spec iteration_options(keyspace()) -> + iteration_options(). iteration_options(Keyspace) -> case keyspace_config(Keyspace) of {emqx_ds_message_storage_bitmask, Config} -> @@ -43,7 +49,7 @@ iteration_options(Keyspace) -> default_iteration_options() end. --spec default_iteration_options() -> emqx_ds_message_storage_bitmask:iteration_options(). +-spec default_iteration_options() -> iteration_options(). default_iteration_options() -> {emqx_ds_message_storage_bitmask, Config} = default_keyspace_config(), maps:get(iteration, Config). @@ -60,7 +66,7 @@ default_keyspace_config() -> } }}. --spec db_options(emqx_ds:keyspace()) -> emqx_ds_storage_layer:db_options(). +-spec db_options(keyspace()) -> emqx_ds_storage_layer:options(). db_options(Keyspace) -> DefaultDBOptions = application:get_env(?APP, default_db_options, []), Keyspaces = application:get_env(?APP, keyspace_config, #{}), diff --git a/apps/emqx_durable_storage/src/emqx_ds_lts.erl b/apps/emqx_durable_storage/src/emqx_ds_lts.erl index d148e8cbc..bcf95852d 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_lts.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_lts.erl @@ -24,7 +24,12 @@ %% Debug: -export([trie_next/3, trie_insert/3, dump_to_dot/2]). --export_type([options/0, static_key/0, trie/0]). +-export_type([ + options/0, + static_key/0, + trie/0, + msg_storage_key/0 +]). -include_lib("stdlib/include/ms_transform.hrl"). diff --git a/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl b/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl index 7a26b696d..a9d904da1 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl @@ -90,7 +90,7 @@ ?enc := emqx_ds_storage_layer:iterator() }. --type message_id() :: emqx_ds_storage_layer:message_id(). +-type message_id() :: emqx_ds:message_id(). -define(batch_messages, 2). @@ -219,7 +219,7 @@ do_store_batch_v1(DB, Shard, #{?tag := ?BATCH, ?batch_messages := Messages}, Opt emqx_ds_storage_layer:store_batch({DB, Shard}, Messages, Options). -spec do_get_streams_v1( - emqx_ds:db(), emqx_ds_replicationi_layer:shard_id(), emqx_ds:topic_filter(), emqx_ds:time() + emqx_ds:db(), emqx_ds_replication_layer:shard_id(), emqx_ds:topic_filter(), emqx_ds:time() ) -> [{integer(), emqx_ds_storage_layer:stream()}]. do_get_streams_v1(DB, Shard, TopicFilter, StartTime) -> @@ -227,7 +227,7 @@ do_get_streams_v1(DB, Shard, TopicFilter, StartTime) -> -spec do_make_iterator_v1( emqx_ds:db(), - emqx_ds_storage_layer:shard_id(), + emqx_ds_replication_layer:shard_id(), emqx_ds_storage_layer:stream(), emqx_ds:topic_filter(), emqx_ds:time() diff --git a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl index 5c451206d..a2fc9dbbf 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl @@ -280,7 +280,7 @@ in_sync_replicas_trans(DB, Shard) -> {error, no_shard} end. --spec set_leader_trans(emqx_ds:ds(), emqx_ds_replication_layer:shard_id(), node()) -> +-spec set_leader_trans(emqx_ds:db(), emqx_ds_replication_layer:shard_id(), node()) -> ok. set_leader_trans(DB, Shard, Node) -> [Record0] = mnesia:wread({?SHARD_TAB, {DB, Shard}}), diff --git a/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl b/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl index 54530f428..99feea77a 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl @@ -26,7 +26,16 @@ %% internal exports: -export([db_dir/1]). --export_type([gen_id/0, generation/0, cf_refs/0, stream/0, iterator/0]). +-export_type([ + gen_id/0, + generation/0, + cf_refs/0, + stream/0, + iterator/0, + shard_id/0, + options/0, + prototype/0 +]). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -64,7 +73,7 @@ ?enc := term() }. -%% Note: this might be stored permanently on a remote node. +%% Note: this might be stred permanently on a remote node. -opaque iterator() :: #{ ?tag := ?IT, @@ -109,17 +118,19 @@ %% Shard (runtime): -type shard() :: shard(generation()). +-type options() :: map(). + %%================================================================================ %% Generation callbacks %%================================================================================ %% Create the new schema given generation id and the options. %% Create rocksdb column families. --callback create(shard_id(), rocksdb:db_handle(), gen_id(), _Options) -> +-callback create(shard_id(), rocksdb:db_handle(), gen_id(), Options :: map()) -> {_Schema, cf_refs()}. %% Open the existing schema --callback open(shard_id(), rocsdb:db_handle(), gen_id(), cf_refs(), _Schema) -> +-callback open(shard_id(), rocksdb:db_handle(), gen_id(), cf_refs(), _Schema) -> _Data. -callback store_batch(shard_id(), _Data, [emqx_types:message()], emqx_ds:message_store_opts()) -> @@ -138,7 +149,7 @@ %% API for the replication layer %%================================================================================ --spec open_shard(shard_id(), emqx_ds:builtin_db_opts()) -> ok. +-spec open_shard(shard_id(), options()) -> ok. open_shard(Shard, Options) -> emqx_ds_storage_layer_sup:ensure_shard(Shard, Options). @@ -215,13 +226,13 @@ next(Shard, Iter = #{?tag := ?IT, ?generation := GenId, ?enc := GenIter0}, Batch -define(REF(ShardId), {via, gproc, {n, l, {?MODULE, ShardId}}}). --spec start_link(shard_id(), emqx_ds:builtin_db_opts()) -> +-spec start_link(shard_id(), options()) -> {ok, pid()}. start_link(Shard = {_, _}, Options) -> gen_server:start_link(?REF(Shard), ?MODULE, {Shard, Options}, []). -record(s, { - shard_id :: emqx_ds:shard_id(), + shard_id :: shard_id(), db :: rocksdb:db_handle(), cf_refs :: cf_refs(), schema :: shard_schema(), @@ -352,7 +363,7 @@ commit_metadata(#s{shard_id = ShardId, schema = Schema, shard = Runtime, db = DB ok = put_schema_persistent(DB, Schema), put_schema_runtime(ShardId, Runtime). --spec rocksdb_open(shard_id(), emqx_ds:builtin_db_opts()) -> +-spec rocksdb_open(shard_id(), options()) -> {ok, rocksdb:db_handle(), cf_refs()} | {error, _TODO}. rocksdb_open(Shard, Options) -> DBOptions = [ diff --git a/apps/emqx_durable_storage/src/emqx_ds_storage_layer_sup.erl b/apps/emqx_durable_storage/src/emqx_ds_storage_layer_sup.erl index c2eee8dcb..fd8cf289f 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_storage_layer_sup.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_storage_layer_sup.erl @@ -30,7 +30,7 @@ start_link() -> start_shard(Shard, Options) -> supervisor:start_child(?SUP, shard_child_spec(Shard, Options)). --spec stop_shard(emqx_ds:shard()) -> ok | {error, _}. +-spec stop_shard(emqx_ds_storage_layer:shard_id()) -> ok | {error, _}. stop_shard(Shard) -> ok = supervisor:terminate_child(?SUP, Shard), ok = supervisor:delete_child(?SUP, Shard). diff --git a/apps/emqx_durable_storage/src/proto/emqx_ds_proto_v1.erl b/apps/emqx_durable_storage/src/proto/emqx_ds_proto_v1.erl index 3b7c36082..5104a417f 100644 --- a/apps/emqx_durable_storage/src/proto/emqx_ds_proto_v1.erl +++ b/apps/emqx_durable_storage/src/proto/emqx_ds_proto_v1.erl @@ -28,8 +28,7 @@ %% API funcions %%================================================================================ --spec drop_db([node()], emqx_ds:db()) -> - [{ok, ok} | erpc:caught_call_exception()]. +-spec drop_db([node()], emqx_ds:db()) -> [emqx_rpc:erpc(ok)]. drop_db(Node, DB) -> erpc:multicall(Node, emqx_ds_replication_layer, do_drop_db_v1, [DB]). @@ -65,7 +64,7 @@ make_iterator(Node, DB, Shard, Stream, TopicFilter, StartTime) -> emqx_ds_storage_layer:iterator(), pos_integer() ) -> - {ok, emqx_ds_storage_layer:iterator(), [emqx_types:messages()]} + {ok, emqx_ds_storage_layer:iterator(), [emqx_types:message()]} | {ok, end_of_stream} | {error, _}. next(Node, DB, Shard, Iter, BatchSize) -> diff --git a/apps/emqx_exhook/src/emqx_exhook_mgr.erl b/apps/emqx_exhook/src/emqx_exhook_mgr.erl index 6ff5350e2..f12fe6522 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mgr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mgr.erl @@ -72,6 +72,10 @@ import_config/1 ]). +-export_type([ + server_name/0 +]). + %% Running servers -type state() :: #{servers := servers()}. diff --git a/apps/emqx_exhook/src/emqx_exhook_sup.erl b/apps/emqx_exhook/src/emqx_exhook_sup.erl index 9a2f07baa..00aa26e18 100644 --- a/apps/emqx_exhook/src/emqx_exhook_sup.erl +++ b/apps/emqx_exhook/src/emqx_exhook_sup.erl @@ -39,6 +39,10 @@ shutdown => Timeout }). +%% TODO: export_type: +%% grpc_client_sup:options/0 +-type grpc_client_sup_options() :: map(). + %%-------------------------------------------------------------------- %% Supervisor APIs & Callbacks %%-------------------------------------------------------------------- @@ -59,7 +63,7 @@ init([]) -> -spec start_grpc_client_channel( binary(), uri_string:uri_string(), - grpc_client_sup:options() + grpc_client_sup_options() ) -> {ok, pid()} | {error, term()}. start_grpc_client_channel(Name, SvrAddr, Options) -> grpc_client_sup:create_channel_pool(Name, SvrAddr, Options). diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src index cb86c1450..e67a4cfa2 100644 --- a/apps/emqx_ft/src/emqx_ft.app.src +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -1,6 +1,6 @@ {application, emqx_ft, [ {description, "EMQX file transfer over MQTT"}, - {vsn, "0.1.9"}, + {vsn, "0.1.10"}, {registered, []}, {mod, {emqx_ft_app, []}}, {applications, [ diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 180528f1f..f5c89ec8b 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -220,7 +220,7 @@ error_desc('SERVICE_UNAVAILABLE') -> roots() -> []. --spec fields(hocon_schema:name()) -> [hoconsc:field()]. +-spec fields(hocon_schema:name()) -> [hocon_schema:field()]. fields(client_id) -> [ {clientid, diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 886dc27f6..edefda22b 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -36,7 +36,7 @@ %% Internal API -export([exporter/1]). --export_type([export/0]). +-export_type([export/0, exporter_conf/0]). -type storage() :: emqx_ft_storage_fs:storage() | undefined. -type transfer() :: emqx_ft:transfer(). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 0102756ca..454cc430c 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -56,6 +56,7 @@ -export_type([filefrag/1]). -export_type([filefrag/0]). -export_type([transferinfo/0]). +-export_type([segmentinfo/0]). -export_type([file_error/0]). @@ -104,7 +105,7 @@ type := 'local', enable := true, segments := segments(), - exporter := emqx_ft_storage_exporter:exporter() + exporter := emqx_ft_storage_exporter:exporter_conf() }. -type file_error() :: diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl index 513872edd..e4aec2433 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl @@ -66,14 +66,14 @@ table(ReaderPid, Bytes) when is_pid(ReaderPid) andalso is_integer(Bytes) andalso end, qlc:table(fun() -> NextFun(ReaderPid) end, []). --spec start_link(pid(), filename:filename()) -> startlink_ret(). +-spec start_link(pid(), file:name()) -> startlink_ret(). start_link(CallerPid, Filename) when is_pid(CallerPid) andalso ?IS_FILENAME(Filename) -> gen_server:start_link(?MODULE, [CallerPid, Filename], []). --spec start_supervised(pid(), filename:filename()) -> startlink_ret(). +-spec start_supervised(pid(), file:name()) -> startlink_ret(). start_supervised(CallerPid, Filename) when is_pid(CallerPid) andalso ?IS_FILENAME(Filename) diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl index 4d77138cf..2225f375f 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl @@ -20,7 +20,7 @@ %% module if it integrated with emqx_gateway_conn module -module(emqx_gateway_channel). --export_type([gen_server_from/0]). +-export_type([gen_server_from/0, channel/0]). -type channel() :: any(). @@ -39,7 +39,7 @@ %% Init %% @doc Initialize the channel state --callback init(emqx_types:conniinfo(), map()) -> channel(). +-callback init(emqx_types:conninfo(), map()) -> channel(). %%-------------------------------------------------------------------- %% Handles @@ -49,8 +49,8 @@ -type gen_server_from() :: {pid(), Tag :: term()}. -type reply() :: - {outgoing, emqx_gateway_frame:packet()} - | {outgoing, [emqx_gateway_frame:packet()]} + {outgoing, emqx_gateway_frame:frame()} + | {outgoing, [emqx_gateway_frame:frame()]} | {event, conn_state() | updated} | {close, Reason :: atom()}. diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index 9f3344e95..d61d086b4 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -73,7 +73,7 @@ %% Parse State parse_state :: emqx_gateway_frame:parse_state(), %% Serialize options - serialize :: emqx_gateway_frame:serialize_opts(), + serialize :: emqx_gateway_frame:serialize_options(), %% Channel State channel :: emqx_gateway_channel:channel(), %% GC State diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_frame.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_frame.erl index 5efd2ee73..5f8038bcf 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_frame.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_frame.erl @@ -22,6 +22,13 @@ %% -module(emqx_gateway_frame). +-export_type([ + parse_state/0, + parse_result/0, + serialize_options/0, + frame/0 +]). + -type parse_state() :: map(). -type frame() :: any(). @@ -32,13 +39,6 @@ -type serialize_options() :: map(). --export_type([ - parse_state/0, - parse_result/0, - serialize_options/0, - frame/0 -]). - %% Callbacks %% @doc Initial the frame parser states diff --git a/apps/emqx_gateway/src/emqx_gateway_registry.erl b/apps/emqx_gateway/src/emqx_gateway_registry.erl index 20a3e1c42..314e974b4 100644 --- a/apps/emqx_gateway/src/emqx_gateway_registry.erl +++ b/apps/emqx_gateway/src/emqx_gateway_registry.erl @@ -42,6 +42,8 @@ code_change/3 ]). +-export_type([descriptor/0]). + -record(state, { reged = #{} :: #{gateway_name() => descriptor()} }). diff --git a/apps/emqx_utils/src/emqx_placeholder.erl b/apps/emqx_utils/src/emqx_placeholder.erl index 90df6003b..dbb2a4e1d 100644 --- a/apps/emqx_utils/src/emqx_placeholder.erl +++ b/apps/emqx_utils/src/emqx_placeholder.erl @@ -46,6 +46,8 @@ quote_mysql/1 ]). +-export_type([tmpl_token/0]). + -define(PH_VAR_THIS, '$this'). %% To match any pattern starts with '$' and followed by '{', and closed by a '}' char: From 50f4aba5cd7d49b85654bde44b3dc0f157261ca4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 8 Dec 2023 20:55:07 +0100 Subject: [PATCH 12/50] fix(dialyzer): batch 3 --- apps/emqx/src/emqx_connection.erl | 2 +- apps/emqx/src/emqx_hocon.erl | 2 +- apps/emqx/src/emqx_hookpoints.erl | 4 ++-- apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl | 2 +- .../src/emqx_limiter/src/emqx_limiter_bucket_ref.erl | 2 +- .../src/emqx_limiter/src/emqx_limiter_container.erl | 4 ++-- apps/emqx/src/emqx_listeners.erl | 2 ++ apps/emqx/src/emqx_metrics.erl | 4 ++-- apps/emqx/src/emqx_persistent_session_ds.erl | 10 +++++++--- .../src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl | 2 +- apps/emqx/src/emqx_router.erl | 2 +- apps/emqx/src/emqx_session.erl | 3 ++- apps/emqx/src/emqx_session_events.erl | 2 +- apps/emqx/src/emqx_sup.erl | 6 +++--- apps/emqx/src/emqx_tls_lib.erl | 11 +++++++---- apps/emqx/src/emqx_tls_psk.erl | 1 + apps/emqx/src/emqx_topic_index.erl | 2 ++ apps/emqx/src/emqx_trace/emqx_trace_formatter.erl | 5 ++++- apps/emqx/src/emqx_trace/emqx_trace_handler.erl | 2 +- apps/emqx_durable_storage/src/emqx_ds.erl | 4 ++++ apps/emqx_machine/src/emqx_machine.app.src | 8 +------- rebar.config.erl | 3 ++- 22 files changed, 49 insertions(+), 34 deletions(-) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 66160ed36..fd7ce4148 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -979,7 +979,7 @@ handle_cast(Req, State) -> %% rate limit -type limiter_type() :: emqx_limiter_container:limiter_type(). --type limiter() :: emqx_limiter_container:limiter(). +-type limiter() :: emqx_limiter_container:container(). -type check_succ_handler() :: fun((any(), list(any()), state()) -> _). diff --git a/apps/emqx/src/emqx_hocon.erl b/apps/emqx/src/emqx_hocon.erl index 68594ad9d..56ac6463f 100644 --- a/apps/emqx/src/emqx_hocon.erl +++ b/apps/emqx/src/emqx_hocon.erl @@ -138,7 +138,7 @@ compact_errors(SchemaModule, Error, Stacktrace) -> }}. %% @doc This is only used in static check scripts in the CI. --spec load_and_check(module(), filename:filename_all()) -> {ok, term()} | {error, any()}. +-spec load_and_check(module(), file:name_all()) -> {ok, term()} | {error, any()}. load_and_check(SchemaModule, File) -> try do_load_and_check(SchemaModule, File) diff --git a/apps/emqx/src/emqx_hookpoints.erl b/apps/emqx/src/emqx_hookpoints.erl index ba125101e..31861b421 100644 --- a/apps/emqx/src/emqx_hookpoints.erl +++ b/apps/emqx/src/emqx_hookpoints.erl @@ -125,12 +125,12 @@ when -callback 'client.subscribe'(emqx_types:clientinfo(), emqx_types:properties(), TopicFilters) -> fold_callback_result(TopicFilters) when - TopicFilters :: list({emqx_topic:topic(), map()}). + TopicFilters :: list({emqx_types:topic(), map()}). -callback 'client.unsubscribe'(emqx_types:clientinfo(), emqx_types:properties(), TopicFilters) -> fold_callback_result(TopicFilters) when - TopicFilters :: list({emqx_topic:topic(), map()}). + TopicFilters :: list({emqx_types:topic(), map()}). -callback 'client.timeout'(_TimerReference :: reference(), _Msg :: term(), Replies) -> fold_callback_result(Replies) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl index 15e838b6e..d53819e90 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl @@ -32,7 +32,7 @@ make_future/1, available/1 ]). --export_type([local_limiter/0, limiter/0]). +-export_type([local_limiter/0, limiter/0, retry_context/1]). %% a token bucket limiter which may or not contains a reference to another limiter, %% and can be used in a client alone diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl index 139564df7..873f70db7 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl @@ -51,7 +51,7 @@ %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- --spec new(counters:countres_ref(), index(), rate()) -> bucket_ref(). +-spec new(counters:counters_ref(), index(), rate()) -> bucket_ref(). new(Counter, Index, Rate) -> #{ counter => Counter, diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl index fb97a9b67..0823a10d7 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl @@ -32,7 +32,7 @@ retry_list/2 ]). --export_type([container/0, check_result/0]). +-export_type([limiter/0, container/0, check_result/0, limiter_type/0]). -type container() :: infinity @@ -51,7 +51,7 @@ -type limiter_id() :: emqx_limiter_schema:limiter_id(). -type limiter_type() :: emqx_limiter_schema:limiter_type(). -type limiter() :: emqx_htb_limiter:limiter(). --type retry_context() :: emqx_htb_limiter:retry_context(). +-type retry_context() :: emqx_htb_limiter:retry_context(limiter()). -type millisecond() :: non_neg_integer(). -type check_result() :: {ok, container()} diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 9abff250b..5cf631ccb 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -63,6 +63,8 @@ -export([certs_dir/2]). -endif. +-export_type([listener_id/0]). + -type listener_id() :: atom() | binary(). -define(ROOT_KEY, listeners). -define(CONF_KEY_PATH, [?ROOT_KEY, '?', '?']). diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index ee86457b0..40427c192 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -72,7 +72,7 @@ %% BACKW: v4.3.0 -export([upgrade_retained_delayed_counter_type/0]). --export_type([metric_idx/0]). +-export_type([metric_idx/0, metric_name/0]). -compile({inline, [inc/1, inc/2, dec/1, dec/2]}). -compile({inline, [inc_recv/1, inc_sent/1]}). @@ -438,7 +438,7 @@ update_counter(Name, Value) -> %% Inc received/sent metrics %%-------------------------------------------------------------------- --spec inc_msg(emqx_types:massage()) -> ok. +-spec inc_msg(emqx_types:message()) -> ok. inc_msg(Msg) -> case Msg#message.qos of 0 -> inc('messages.qos0.received'); diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index 37385c12c..a1db2e933 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -85,6 +85,12 @@ ]). -endif. +-export_type([ + id/0, + subscription_id/0, + session/0 +]). + %% Currently, this is the clientid. We avoid `emqx_types:clientid()' because that can be %% an atom, in theory (?). -type id() :: binary(). @@ -145,8 +151,6 @@ (NOW_MS >= LAST_ALIVE_AT + EI)) ). --export_type([id/0]). - %% -spec create(clientinfo(), conninfo(), emqx_session:conf()) -> @@ -243,7 +247,7 @@ stats(Session) -> info(?STATS_KEYS, Session). %% Debug/troubleshooting --spec print_session(emqx_types:client_id()) -> map() | undefined. +-spec print_session(emqx_types:clientid()) -> map() | undefined. print_session(ClientId) -> catch ro_transaction( fun() -> diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl b/apps/emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl index 30820b126..cb6296b59 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl @@ -24,7 +24,7 @@ dest :: emqx_persistent_session_ds:id() }). -record(ps_routeidx, { - entry :: emqx_topic_index:key(emqx_persistent_session_ds_router:dest()), + entry :: '$1' | emqx_topic_index:key(emqx_persistent_session_ds_router:dest()), unused = [] :: nil() }). diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 1aebb1b21..3b92d4f1c 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -91,7 +91,7 @@ -type dest() :: node() | {group(), node()}. -record(routeidx, { - entry :: emqx_topic_index:key(dest()), + entry :: '$1' | emqx_topic_index:key(dest()), unused = [] :: nil() }). diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index fa9469a61..c08109fe8 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -111,6 +111,7 @@ t/0, conf/0, conninfo/0, + clientinfo/0, reply/0, replies/0, common_timer_name/0, @@ -499,7 +500,7 @@ cancel_timer(Name, Timers0) -> %%-------------------------------------------------------------------- --spec disconnect(clientinfo(), eqmx_types:conninfo(), t()) -> +-spec disconnect(clientinfo(), conninfo(), t()) -> {idle | shutdown, t()}. disconnect(_ClientInfo, ConnInfo, Session) -> ?IMPL(Session):disconnect(Session, ConnInfo). diff --git a/apps/emqx/src/emqx_session_events.erl b/apps/emqx/src/emqx_session_events.erl index b04dd2044..f46144020 100644 --- a/apps/emqx/src/emqx_session_events.erl +++ b/apps/emqx/src/emqx_session_events.erl @@ -34,7 +34,7 @@ %% --spec handle_event(emqx_session:client_info(), event()) -> +-spec handle_event(emqx_session:clientinfo(), event()) -> ok. handle_event(ClientInfo, {expired, Msg}) -> ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, expired]), diff --git a/apps/emqx/src/emqx_sup.erl b/apps/emqx/src/emqx_sup.erl index 65742a234..24314362c 100644 --- a/apps/emqx/src/emqx_sup.erl +++ b/apps/emqx/src/emqx_sup.erl @@ -30,8 +30,8 @@ -export([init/1]). -type startchild_ret() :: - {ok, supervisor:child()} - | {ok, supervisor:child(), term()} + {ok, pid()} + | {ok, pid(), term()} | {error, term()}. -define(SUP, ?MODULE). @@ -52,7 +52,7 @@ start_child(ChildSpec) when is_map(ChildSpec) -> start_child(Mod, Type) -> start_child(child_spec(Mod, Type)). --spec stop_child(supervisor:child_id()) -> ok | {error, term()}. +-spec stop_child(atom()) -> ok | {error, term()}. stop_child(ChildId) -> case supervisor:terminate_child(?SUP, ChildId) of ok -> supervisor:delete_child(?SUP, ChildId); diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 0b9bfe805..32c572b42 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -44,6 +44,9 @@ to_client_opts/2 ]). +%% ssl:tls_version/0 is not exported. +-type tls_version() :: tlsv1 | 'tlsv1.1' | 'tlsv1.2' | 'tlsv1.3'. + -include("logger.hrl"). -define(IS_TRUE(Val), ((Val =:= true) orelse (Val =:= <<"true">>))). @@ -123,8 +126,8 @@ %% @doc Validate a given list of desired tls versions. %% raise an error exception if non of them are available. %% The input list can be a string/binary of comma separated versions. --spec integral_versions(tls | dtls, undefined | string() | binary() | [ssl:tls_version()]) -> - [ssl:tls_version()]. +-spec integral_versions(tls | dtls, undefined | string() | binary() | [tls_version()]) -> + [tls_version()]. integral_versions(Type, undefined) -> available_versions(Type); integral_versions(Type, []) -> @@ -164,7 +167,7 @@ all_ciphers() -> all_ciphers(available_versions(all)). %% @hidden Return a list of (openssl string format) cipher suites. --spec all_ciphers([ssl:tls_version()]) -> [string()]. +-spec all_ciphers([tls_version()]) -> [string()]. all_ciphers(['tlsv1.3']) -> %% When it's only tlsv1.3 wanted, use 'exclusive' here %% because 'all' returns legacy cipher suites too, @@ -212,7 +215,7 @@ do_selected_ciphers(_) -> ?SELECTED_CIPHERS. %% @doc Ensure version & cipher-suites integrity. --spec integral_ciphers([ssl:tls_version()], binary() | string() | [string()]) -> [string()]. +-spec integral_ciphers([tls_version()], binary() | string() | [string()]) -> [string()]. integral_ciphers(Versions, Ciphers) when Ciphers =:= [] orelse Ciphers =:= undefined -> %% not configured integral_ciphers(Versions, selected_ciphers(Versions)); diff --git a/apps/emqx/src/emqx_tls_psk.erl b/apps/emqx/src/emqx_tls_psk.erl index 871c41368..1041e6ee0 100644 --- a/apps/emqx/src/emqx_tls_psk.erl +++ b/apps/emqx/src/emqx_tls_psk.erl @@ -20,6 +20,7 @@ %% SSL PSK Callbacks -export([lookup/3]). +-export_type([psk_identity/0]). -type psk_identity() :: string(). -type psk_user_state() :: term(). diff --git a/apps/emqx/src/emqx_topic_index.erl b/apps/emqx/src/emqx_topic_index.erl index eaedb2e53..acf10c11c 100644 --- a/apps/emqx/src/emqx_topic_index.erl +++ b/apps/emqx/src/emqx_topic_index.erl @@ -30,6 +30,8 @@ -export([get_topic/1]). -export([get_record/2]). +-export_type([key/1]). + -type key(ID) :: emqx_trie_search:key(ID). -type match(ID) :: key(ID). -type words() :: emqx_trie_search:words(). diff --git a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl index ae2596808..49b7e9890 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl @@ -19,11 +19,14 @@ -export([format/2]). -export([format_meta_map/1]). +%% logger_formatter:config/0 is not exported. +-type config() :: map(). + %%%----------------------------------------------------------------- %%% API -spec format(LogEvent, Config) -> unicode:chardata() when LogEvent :: logger:log_event(), - Config :: logger:config(). + Config :: config(). format( #{level := debug, meta := Meta = #{trace_tag := Tag}, msg := Msg}, #{payload_encode := PEncode} diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl index 528bc4d42..dcb9ce0c9 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl @@ -125,7 +125,7 @@ uninstall(HandlerId) -> name => binary(), type => topic | clientid | ip_address, id => atom(), - filter => emqx_types:topic() | emqx_types:clienetid() | emqx_trace:ip_address(), + filter => emqx_types:topic() | emqx_types:clientid() | emqx_trace:ip_address(), level => logger:level(), dst => file:filename() | console | unknown } diff --git a/apps/emqx_durable_storage/src/emqx_ds.erl b/apps/emqx_durable_storage/src/emqx_ds.erl index 4516a3c86..1d18c5d73 100644 --- a/apps/emqx_durable_storage/src/emqx_ds.erl +++ b/apps/emqx_durable_storage/src/emqx_ds.erl @@ -42,6 +42,7 @@ stream/0, stream_rank/0, iterator/0, + iterator_id/0, message_id/0, message_store_opts/0, next_result/1, next_result/0, @@ -67,6 +68,9 @@ -type stream_rank() :: {term(), integer()}. +%% TODO: Not implemented +-type iterator_id() :: term(). + -opaque iterator() :: ds_specific_iterator(). -opaque stream() :: ds_specific_stream(). diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index e3b37fc23..6d7012313 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -6,13 +6,7 @@ {vsn, "0.2.17"}, {modules, []}, {registered, []}, - {applications, [kernel, stdlib, emqx_ctl, covertool]}, - %% system_monitor is loaded but not booted, - %% emqx_machine.erl makes the decision when to start - %% the app after certain config injection. - %% it's a included_application because otherwise dialyzer - %% would report unknown functions - {included_applications, [system_monitor]}, + {applications, [kernel, stdlib, emqx_ctl]}, {mod, {emqx_machine_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/rebar.config.erl b/rebar.config.erl index 7ab693979..6958ad784 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -562,7 +562,8 @@ dialyzer(Config) -> AppsToExclude = AppNames -- KnownApps, Extra = - [bcrypt || provide_bcrypt_dep()] ++ + [os_mon, system_monitor, covertool] ++ + [bcrypt || provide_bcrypt_dep()] ++ [jq || is_jq_supported()] ++ [quicer || is_quicer_supported()], NewDialyzerConfig = From 68da627b4dc80fe1e67b12a80cfe959e92c133e1 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 6 Dec 2023 15:28:28 +0800 Subject: [PATCH 13/50] feat(channel): add peerport field in ClientInfo --- apps/emqx/src/emqx_channel.erl | 3 ++- apps/emqx/test/emqx_channel_SUITE.erl | 19 ++++++++++++++++++- apps/emqx/test/emqx_proper_types.erl | 1 + 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 4662eaee5..92715cf80 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -213,7 +213,7 @@ caps(#channel{clientinfo = #{zone := Zone}}) -> -spec init(emqx_types:conninfo(), opts()) -> channel(). init( ConnInfo = #{ - peername := {PeerHost, _Port}, + peername := {PeerHost, PeerPort}, sockname := {_Host, SockPort} }, #{ @@ -237,6 +237,7 @@ init( listener => ListenerId, protocol => Protocol, peerhost => PeerHost, + peerport => PeerPort, sockport => SockPort, clientid => undefined, username => undefined, diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index c6b4c0518..ca038ac85 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -72,6 +72,22 @@ t_chan_info(_) -> conn_state := connected, clientinfo := ClientInfo } = emqx_channel:info(channel()), + ?assertMatch( + #{ + zone := default, + listener := {tcp, default}, + protocol := mqtt, + peerhost := {127, 0, 0, 1}, + peerport := 3456, + sockport := 1883, + clientid := <<"clientid">>, + username := <<"username">>, + is_superuser := false, + is_bridge := false, + mountpoint := undefined + }, + ClientInfo + ), ?assertEqual(clientinfo(), ClientInfo). t_chan_caps(_) -> @@ -1063,7 +1079,8 @@ clientinfo(InitProps) -> listener => {tcp, default}, protocol => mqtt, peerhost => {127, 0, 0, 1}, - sockport => 3456, + peerport => 3456, + sockport => 1883, clientid => <<"clientid">>, username => <<"username">>, is_superuser => false, diff --git a/apps/emqx/test/emqx_proper_types.erl b/apps/emqx/test/emqx_proper_types.erl index 6c2ad56f9..243a39007 100644 --- a/apps/emqx/test/emqx_proper_types.erl +++ b/apps/emqx/test/emqx_proper_types.erl @@ -108,6 +108,7 @@ clientinfo() -> {zone, zone()}, {protocol, protocol()}, {peerhost, ip()}, + {peerport, port()}, {sockport, port()}, {clientid, clientid()}, {username, username()}, From 46201a8796d4ce8e333c8f7d4e360157f7f430e3 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 6 Dec 2023 10:54:55 +0800 Subject: [PATCH 14/50] feat(exhook): provide the `peerport` field - both in `ConnInfo` and `ClientInfo` --- apps/emqx_exhook/priv/protos/exhook.proto | 4 ++++ apps/emqx_exhook/src/emqx_exhook_handler.erl | 5 ++++- apps/emqx_exhook/test/emqx_exhook_SUITE.erl | 2 ++ apps/emqx_exhook/test/props/prop_exhook_hooks.erl | 5 +++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/apps/emqx_exhook/priv/protos/exhook.proto b/apps/emqx_exhook/priv/protos/exhook.proto index e5d7b3606..6f6b860af 100644 --- a/apps/emqx_exhook/priv/protos/exhook.proto +++ b/apps/emqx_exhook/priv/protos/exhook.proto @@ -368,6 +368,8 @@ message ConnInfo { string proto_ver = 7; uint32 keepalive = 8; + + uint32 peerport = 9; } message ClientInfo { @@ -397,6 +399,8 @@ message ClientInfo { // subject of client TLS cert string dn = 12; + + uint32 peerport = 13; } message Message { diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index 2bcb91b12..f3dfa111c 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -294,7 +294,7 @@ conninfo( ConnInfo = #{ clientid := ClientId, - peername := {Peerhost, _}, + peername := {Peerhost, PeerPort}, sockname := {_, SockPort} } ) -> @@ -307,6 +307,7 @@ conninfo( clientid => ClientId, username => maybe(Username), peerhost => ntoa(Peerhost), + peerport => PeerPort, sockport => SockPort, proto_name => ProtoName, proto_ver => stringfy(ProtoVer), @@ -319,6 +320,7 @@ clientinfo( clientid := ClientId, username := Username, peerhost := PeerHost, + peerport := PeerPort, sockport := SockPort, protocol := Protocol, mountpoint := Mountpoiont @@ -330,6 +332,7 @@ clientinfo( username => maybe(Username), password => maybe(maps:get(password, ClientInfo, undefined)), peerhost => ntoa(PeerHost), + peerport => PeerPort, sockport => SockPort, protocol => stringfy(Protocol), mountpoint => maybe(Mountpoiont), diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl index 3da73c11a..ffe932449 100644 --- a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl @@ -118,6 +118,7 @@ t_access_failed_if_no_server_running(Config) -> clientid => <<"user-id-1">>, username => <<"usera">>, peerhost => {127, 0, 0, 1}, + peerport => 3456, sockport => 1883, protocol => mqtt, mountpoint => undefined @@ -301,6 +302,7 @@ t_simulated_handler(_) -> clientid => <<"user-id-1">>, username => <<"usera">>, peerhost => {127, 0, 0, 1}, + peerport => 3456, sockport => 1883, protocol => mqtt, mountpoint => undefined diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl index cf48fff80..827205d1d 100644 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl @@ -496,6 +496,9 @@ nodestr() -> peerhost(#{peername := {Host, _}}) -> ntoa(Host). +peerport(#{peername := {_, Port}}) -> + Port. + sockport(#{sockname := {_, Port}}) -> Port. @@ -564,6 +567,7 @@ from_conninfo(ConnInfo) -> clientid => maps:get(clientid, ConnInfo), username => maybe(maps:get(username, ConnInfo, <<>>)), peerhost => peerhost(ConnInfo), + peerport => peerport(ConnInfo), sockport => sockport(ConnInfo), proto_name => maps:get(proto_name, ConnInfo), proto_ver => stringfy(maps:get(proto_ver, ConnInfo)), @@ -577,6 +581,7 @@ from_clientinfo(ClientInfo) -> username => maybe(maps:get(username, ClientInfo, <<>>)), password => maybe(maps:get(password, ClientInfo, <<>>)), peerhost => ntoa(maps:get(peerhost, ClientInfo)), + peerport => maps:get(peerport, ClientInfo), sockport => maps:get(sockport, ClientInfo), protocol => stringfy(maps:get(protocol, ClientInfo)), mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), From cfe3b2dcee6a878c2bdb7f5d7d0eb200aa5f05e3 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 6 Dec 2023 18:02:06 +0800 Subject: [PATCH 15/50] feat(exhook): subopts in on_client_subscribe/on_client_unsubscribe --- apps/emqx_exhook/priv/protos/exhook.proto | 27 ++++++++++++++++--- apps/emqx_exhook/src/emqx_exhook_handler.erl | 16 +++++++---- .../test/props/prop_exhook_hooks.erl | 5 +++- 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/apps/emqx_exhook/priv/protos/exhook.proto b/apps/emqx_exhook/priv/protos/exhook.proto index 6f6b860af..27ce25661 100644 --- a/apps/emqx_exhook/priv/protos/exhook.proto +++ b/apps/emqx_exhook/priv/protos/exhook.proto @@ -148,6 +148,9 @@ message ClientAuthorizeRequest { AuthorizeReqType type = 2; + // In ClientAuthorizeRequest. + // Only "real-topic" will be serialized in gRPC request when shared-sub. + // For example, when client subscribes to `$share/group/t/1`, the real topic is `t/1`. string topic = 3; bool result = 4; @@ -456,7 +459,14 @@ message TopicFilter { string name = 1; - uint32 qos = 2; + // Deprecated + // Since EMQX 5.4.0, we have deprecated the 'qos' field in the `TopicFilter` structure. + // A new field named 'subopts,' has been added to encompass all subscription options. + // Please see the `SubOpts` structure for details. + reserved 2; + reserved "qos"; + + SubOpts subopts = 3; } message SubOpts { @@ -464,11 +474,20 @@ message SubOpts { // The QoS level uint32 qos = 1; - // deprecated + // Deprecated reserved 2; reserved "share"; - // The group name for shared subscription - // string share = 2; + // Since EMQX 5.4.0, we have deprecated the 'share' field in the `SubOpts` structure. + // The group name of shared subscription will be serialized with topic. + // hooks: + // "client.subscribe": + // ClientSubscribeRequest.TopicFilter.name = "$share/group/topic/1" + // "client.unsubscribe": + // ClientUnsubscribeRequest.TopicFilter.name = "$share/group/topic/1" + // "session.subscribed": + // SessionSubscribedRequest.topic = "$share/group/topic/1" + // "session.unsubscribed": + // SessionUnsubscribedRequest.topic = "$share/group/topic/1" // The Retain Handling option (MQTT v5.0) // diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index f3dfa111c..fe6af653b 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -192,7 +192,7 @@ on_session_subscribed(ClientInfo, Topic, SubOpts) -> Req = #{ clientinfo => clientinfo(ClientInfo), topic => emqx_topic:maybe_format_share(Topic), - subopts => maps:with([qos, rh, rap, nl], SubOpts) + subopts => subopts(SubOpts) }, cast('session.subscribed', Req). @@ -200,6 +200,7 @@ on_session_unsubscribed(ClientInfo, Topic, _SubOpts) -> Req = #{ clientinfo => clientinfo(ClientInfo), topic => emqx_topic:maybe_format_share(Topic) + %% no subopts when unsub }, cast('session.unsubscribed', Req). @@ -416,14 +417,19 @@ enrich_header(Headers, Message) -> end. topicfilters(Tfs) when is_list(Tfs) -> - GetQos = fun(SubOpts) -> - maps:get(qos, SubOpts, 0) - end, [ - #{name => emqx_topic:maybe_format_share(Topic), qos => GetQos(SubOpts)} + #{name => emqx_topic:maybe_format_share(Topic), subopts => subopts(SubOpts)} || {Topic, SubOpts} <- Tfs ]. +subopts(SubOpts) -> + #{ + qos => maps:get(qos, SubOpts, 0), + rh => maps:get(rh, SubOpts, 0), + rap => maps:get(rap, SubOpts, 0), + nl => maps:get(nl, SubOpts, 0) + }. + ntoa({0, 0, 0, 0, 0, 16#ffff, AB, CD}) -> list_to_binary(inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256})); ntoa(IP) -> diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl index 827205d1d..041091e27 100644 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl @@ -530,7 +530,10 @@ properties(M) when is_map(M) -> ). topicfilters(Tfs) when is_list(Tfs) -> - [#{name => Topic, qos => Qos} || {Topic, #{qos := Qos}} <- Tfs]. + [ + #{name => emqx_topic:maybe_format_share(Topic), subopts => subopts(SubOpts)} + || {Topic, SubOpts} <- Tfs + ]. %% @private stringfy(Term) when is_binary(Term) -> From 47901c9fed09775e564de4057e4f8cac7d016551 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Sun, 10 Dec 2023 22:21:42 +0800 Subject: [PATCH 16/50] fix(exhook): `client.authorize` hook always uses real-topic See: emqx_channel:do_check_sub_authzs/3, line: 1895 --- apps/emqx_exhook/src/emqx_exhook_handler.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index fe6af653b..5c1d18c17 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -143,7 +143,7 @@ on_client_authorize(ClientInfo, Action, Topic, Result) -> Req = #{ clientinfo => clientinfo(ClientInfo), type => Type, - topic => emqx_topic:maybe_format_share(Topic), + topic => emqx_topic:get_shared_real_topic(Topic), result => Bool }, case From fa4d73835be1ce9fec7d7d9d9238cebaa0bda8d4 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Sun, 10 Dec 2023 23:15:55 +0800 Subject: [PATCH 17/50] chore: bump changes --- changes/feat-12114.en.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changes/feat-12114.en.md diff --git a/changes/feat-12114.en.md b/changes/feat-12114.en.md new file mode 100644 index 000000000..7e1b5536b --- /dev/null +++ b/changes/feat-12114.en.md @@ -0,0 +1,6 @@ +Added the `peerport` field to ClientInfo. +Added the `peerport` field to the messages `ClientInfo` and `ConnInfo` in ExHook. + +## Breaking changes +* ExHook Proto changed. The `qos` field in message `TopicFilter` was deprecated. + ExHook Server will now receive full subscription options: `qos`, `rh`, `rap`, `nl` in message `SubOpts` From 525a9e0adfc42ddfa131371be20e30ad30c86157 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 11 Dec 2023 14:49:32 +0800 Subject: [PATCH 18/50] fix: only return recommend setting when get prometheus api --- .../src/emqx_dashboard_listener.erl | 5 ++- .../src/emqx_prometheus_api.erl | 24 ++++++++--- .../src/emqx_prometheus_config.erl | 6 ++- .../test/emqx_prometheus_SUITE.erl | 2 +- .../test/emqx_prometheus_api_SUITE.erl | 43 ++++++++++++------- 5 files changed, 55 insertions(+), 25 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl index 5d5062948..6c4b84433 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl @@ -24,6 +24,7 @@ -export([add_handler/0, remove_handler/0]). -export([pre_config_update/3, post_config_update/5]). -export([regenerate_minirest_dispatch/0]). +-export([delay_job/1]). -behaviour(gen_server). @@ -68,7 +69,7 @@ handle_call(_Request, _From, State) -> handle_cast(_Request, State) -> {noreply, State, hibernate}. -handle_info(i18n_lang_changed, _State) -> +handle_info(regenerate, _State) -> NewState = regenerate_minirest_dispatch(), {noreply, NewState, hibernate}; handle_info({update_listeners, OldListeners, NewListeners}, _State) -> @@ -146,7 +147,7 @@ remove_sensitive_data(Conf0) -> end. post_config_update(_, {change_i18n_lang, _}, _NewConf, _OldConf, _AppEnvs) -> - delay_job(i18n_lang_changed); + delay_job(regenerate); post_config_update(_, _Req, NewConf, OldConf, _AppEnvs) -> OldHttp = get_listener(http, OldConf), OldHttps = get_listener(https, OldConf), diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index b08cd8388..44e0fac16 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -50,15 +50,15 @@ schema("/prometheus") -> description => ?DESC(get_prom_conf_info), tags => ?TAGS, responses => - #{200 => prometheus_setting_schema()} + #{200 => prometheus_setting_response()} }, put => #{ description => ?DESC(update_prom_conf_info), tags => ?TAGS, - 'requestBody' => prometheus_setting_schema(), + 'requestBody' => prometheus_setting_request(), responses => - #{200 => prometheus_setting_schema()} + #{200 => prometheus_setting_response()} } }; schema("/prometheus/stats") -> @@ -84,7 +84,13 @@ security() -> %%-------------------------------------------------------------------- setting(get, _Params) -> - {200, emqx:get_raw_config([<<"prometheus">>], #{})}; + Raw = emqx:get_raw_config([<<"prometheus">>], #{}), + Conf = + case emqx_prometheus_schema:is_recommend_type(Raw) of + true -> Raw; + false -> emqx_prometheus_config:to_recommend_type(Raw) + end, + {200, Conf}; setting(put, #{body := Body}) -> case emqx_prometheus_config:update(Body) of {ok, NewConfig} -> @@ -112,7 +118,7 @@ stats(get, #{headers := Headers}) -> %% Internal funcs %%-------------------------------------------------------------------- -prometheus_setting_schema() -> +prometheus_setting_request() -> [{prometheus, #{type := Setting}}] = emqx_prometheus_schema:roots(), emqx_dashboard_swagger:schema_with_examples( Setting, @@ -122,6 +128,14 @@ prometheus_setting_schema() -> ] ). +%% Always return recommend setting +prometheus_setting_response() -> + {_, #{value := Example}} = recommend_setting_example(), + emqx_dashboard_swagger:schema_with_example( + ?R_REF(emqx_prometheus_schema, recommend_setting), + Example + ). + legacy_setting_example() -> Summary = <<"legacy_deprecated_setting">>, {Summary, #{ diff --git a/apps/emqx_prometheus/src/emqx_prometheus_config.erl b/apps/emqx_prometheus/src/emqx_prometheus_config.erl index 66d53f2db..a24b52537 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_config.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_config.erl @@ -23,6 +23,7 @@ -export([pre_config_update/3, post_config_update/5]). -export([update/1]). -export([conf/0, is_push_gateway_server_enabled/1]). +-export([to_recommend_type/1]). update(Config) -> case @@ -60,7 +61,8 @@ pre_config_update(?PROMETHEUS, MergeConf, OriginConf) -> to_recommend_type(Conf) -> #{ <<"push_gateway">> => to_push_gateway(Conf), - <<"collectors">> => to_collectors(Conf) + <<"collectors">> => to_collectors(Conf), + <<"enable_basic_auth">> => false }. to_push_gateway(Conf) -> @@ -123,7 +125,7 @@ update_push_gateway(Prometheus) -> end. update_auth(#{enable_basic_auth := New}, #{enable_basic_auth := Old}) when New =/= Old -> - emqx_dashboard_listener:regenerate_minirest_dispatch(), + emqx_dashboard_listener:delay_job(regenerate), ok; update_auth(_, _) -> ok. diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl index c50813938..496919b10 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl @@ -78,7 +78,7 @@ groups() -> ]. suite() -> - [{timetrap, {seconds, 30}}]. + [{timetrap, {seconds, 60}}]. common_tests() -> emqx_common_test_helpers:all(?MODULE). diff --git a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl index ad2b3d5ec..cc20e60c7 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl @@ -80,29 +80,42 @@ set_special_configs(_App, _) -> %%-------------------------------------------------------------------- %% Cases %%-------------------------------------------------------------------- +%% we return recommend config for prometheus even if prometheus is legacy. t_legacy_prometheus_api(_) -> Path = emqx_mgmt_api_test_util:api_path(["prometheus"]), Auth = emqx_mgmt_api_test_util:auth_header_(), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth), + OldConf = emqx:get_raw_config([prometheus]), Conf = emqx_utils_json:decode(Response, [return_maps]), + %% Always return new config. ?assertMatch( #{ - <<"push_gateway_server">> := _, - <<"interval">> := _, - <<"enable">> := _, - <<"vm_statistics_collector">> := _, - <<"vm_system_info_collector">> := _, - <<"vm_memory_collector">> := _, - <<"vm_msacc_collector">> := _, - <<"headers">> := _ + <<"collectors">> := + #{ + <<"mnesia">> := <<"disabled">>, + <<"vm_dist">> := <<"disabled">>, + <<"vm_memory">> := <<"disabled">>, + <<"vm_msacc">> := <<"disabled">>, + <<"vm_statistics">> := <<"disabled">>, + <<"vm_system_info">> := <<"disabled">> + }, + <<"enable_basic_auth">> := false, + <<"push_gateway">> := + #{ + <<"enable">> := true, + <<"headers">> := #{<<"Authorization">> := <<"some-authz-tokens">>}, + <<"interval">> := <<"1s">>, + <<"job_name">> := <<"${name}~${host}">>, + <<"url">> := <<"http://127.0.0.1:9091">> + } }, Conf ), - #{<<"enable">> := Enable} = Conf, + #{<<"push_gateway">> := #{<<"enable">> := Enable}} = Conf, ?assertEqual(Enable, undefined =/= erlang:whereis(emqx_prometheus)), - NewConf = Conf#{ + NewConf = OldConf#{ <<"interval">> => <<"2s">>, <<"vm_statistics_collector">> => <<"enabled">>, <<"headers">> => #{ @@ -113,7 +126,7 @@ t_legacy_prometheus_api(_) -> {ok, Response2} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf), Conf2 = emqx_utils_json:decode(Response2, [return_maps]), - ?assertMatch(NewConf, Conf2), + ?assertEqual(NewConf, Conf2), EnvCollectors = application:get_env(prometheus, collectors, []), PromCollectors = prometheus_registry:collectors(default), @@ -153,11 +166,11 @@ t_legacy_prometheus_api(_) -> emqx_config:get([prometheus]) ), - NewConf1 = Conf#{<<"enable">> => (not Enable)}, + NewConf1 = OldConf#{<<"enable">> => (not Enable)}, {ok, _Response3} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf1), ?assertEqual((not Enable), undefined =/= erlang:whereis(emqx_prometheus)), - ConfWithoutScheme = Conf#{<<"push_gateway_server">> => "127.0.0.1:8081"}, + ConfWithoutScheme = OldConf#{<<"push_gateway_server">> => "127.0.0.1:8081"}, ?assertMatch( {error, {"HTTP/1.1", 400, _}}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, ConfWithoutScheme) @@ -267,8 +280,7 @@ t_stats_no_auth_api(_) -> %% undefined is legacy prometheus case emqx:get_config([prometheus, enable_basic_auth], undefined) of true -> - {ok, _} = emqx:update_config([prometheus, enable_basic_auth], false), - emqx_dashboard_listener:regenerate_minirest_dispatch(); + {ok, _} = emqx:update_config([prometheus, enable_basic_auth], false); _ -> ok end, @@ -278,6 +290,7 @@ t_stats_no_auth_api(_) -> t_stats_auth_api(_) -> {ok, _} = emqx:update_config([prometheus, enable_basic_auth], true), + emqx_dashboard_listener:regenerate_minirest_dispatch(), Auth = emqx_mgmt_api_test_util:auth_header_(), JsonAuth = [{"accept", "application/json"}, Auth], request_stats(JsonAuth, Auth), From 0595db39116a624a84cfcee718fdc211e920457c Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 11 Dec 2023 14:55:04 +0800 Subject: [PATCH 19/50] chore: update redis bridge example --- apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl index 2169aa1f3..b1a27d1ce 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl @@ -248,20 +248,19 @@ connector_example(RedisType, post) -> maps:merge( connector_example(RedisType, put), #{ - type => <<"redis_single_producer">>, + type => <<"redis">>, name => <<"my_connector">> } ); connector_example(RedisType, put) -> #{ enable => true, - desc => <<"My redis ", (atom_to_binary(RedisType))/binary, " connector">>, + description => <<"My redis ", (atom_to_binary(RedisType))/binary, " connector">>, parameters => connector_parameter(RedisType), pool_size => 8, database => 1, username => <<"test">>, password => <<"******">>, - auto_reconnect => true, ssl => #{enable => false} }. From 7b609282887a1227782d72e497335085e240f2e6 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 11 Dec 2023 15:01:12 +0800 Subject: [PATCH 20/50] fix(api): endpoint `/v5/topics` with invalid topic filter --- .../src/emqx_mgmt_api_topics.erl | 4 +++- .../test/emqx_mgmt_api_topics_SUITE.erl | 17 +++++++++++++++++ changes/fix-12141.en.md | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 changes/fix-12141.en.md diff --git a/apps/emqx_management/src/emqx_mgmt_api_topics.erl b/apps/emqx_management/src/emqx_mgmt_api_topics.erl index c1d5f8e74..02d4461e9 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_topics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_topics.erl @@ -115,7 +115,9 @@ do_list(Params) -> {200, format_list_response(Pager, QResult)} catch throw:{error, page_limit_invalid} -> - {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}} + {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}}; + error:{invalid_topic_filter, _} -> + {400, #{code => <<"INVALID_PARAMTER">>, message => <<"topic_filter_invalid">>}} end. lookup(#{topic := Topic}) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl index 40ca7aa91..d2f0838e0 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl @@ -155,3 +155,20 @@ t_shared_topics(_Configs) -> ), ok = emqtt:stop(Client). + +t_shared_topics_invalid(_Config) -> + %% no real topic + InvalidShareTopicFilter = <<"$share/group">>, + Path = emqx_mgmt_api_test_util:api_path(["topics"]), + QS = uri_string:compose_query([ + {"topic", InvalidShareTopicFilter}, + {"node", atom_to_list(node())} + ]), + Headers = emqx_mgmt_api_test_util:auth_header_(), + {error, {{_, 400, _}, _RespHeaders, Body}} = emqx_mgmt_api_test_util:request_api( + get, Path, QS, Headers, [], #{return_all => true} + ), + ?assertMatch( + #{<<"code">> := <<"INVALID_PARAMTER">>, <<"message">> := <<"topic_filter_invalid">>}, + emqx_utils_json:decode(Body, [return_maps]) + ). diff --git a/changes/fix-12141.en.md b/changes/fix-12141.en.md new file mode 100644 index 000000000..9611756ad --- /dev/null +++ b/changes/fix-12141.en.md @@ -0,0 +1 @@ +Fixed API endpoint `/v5/topics` to return `InternalError` with HTTP status 500 by invalid topic filter. From 8c25371ff3bd46977a7082b92d06512a13412f0c Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 11 Dec 2023 15:08:38 +0800 Subject: [PATCH 21/50] chore: bump gitignore for emqx standalone common test SUITES --- apps/emqx/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 apps/emqx/.gitignore diff --git a/apps/emqx/.gitignore b/apps/emqx/.gitignore new file mode 100644 index 000000000..e33ea388e --- /dev/null +++ b/apps/emqx/.gitignore @@ -0,0 +1,2 @@ +# See: emqx_common_test_helpers:copy_acl_conf/0 +etc/acl.conf From d560366c144a97d03257f0458749e149ce422404 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 11 Dec 2023 08:42:35 +0100 Subject: [PATCH 22/50] test: fix some compile warnings --- apps/emqx/src/emqx_session.erl | 8 ++++++++ .../test/emqx_bridge_hstreamdb_SUITE.erl | 1 - apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl | 4 +++- apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl | 2 ++ .../test/props/emqx_ds_message_storage_bitmask_shim.erl | 3 --- apps/emqx_durable_storage/test/props/payload_gen.erl | 6 +++--- apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl | 2 +- 7 files changed, 17 insertions(+), 9 deletions(-) diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index c08109fe8..7604c2ba5 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -175,11 +175,19 @@ %% Behaviour %% ------------------------------------------------------------------- +-if(?OTP_RELEASE < 26). +-callback create(clientinfo(), conninfo(), conf()) -> + term(). +-callback open(clientinfo(), conninfo(), conf()) -> + term(). +-callback destroy(t() | clientinfo()) -> ok. +-else. -callback create(clientinfo(), conninfo(), conf()) -> t(). -callback open(clientinfo(), conninfo(), conf()) -> {_IsPresent :: true, t(), _ReplayContext} | false. -callback destroy(t() | clientinfo()) -> ok. +-endif. %%-------------------------------------------------------------------- %% Create a Session diff --git a/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl index a9232d5fe..4d165c03d 100644 --- a/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl +++ b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl @@ -262,7 +262,6 @@ t_write_failure(Config) -> ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), QueryMode = ?config(query_mode, Config), - EnableBatch = ?config(enable_batch, Config), Data = rand_data(), {{ok, _}, {ok, _}} = ?wait_async_action( diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index 14d6f48b7..9b32b0d98 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -19,7 +19,7 @@ -behaviour(hocon_schema). %% API --export([paths/0, api_spec/0, schema/1, fields/1]). +-export([paths/0, api_spec/0, schema/1, namespace/0, fields/1]). -export([init_per_suite/1, end_per_suite/1]). -export([t_in_path/1, t_in_query/1, t_in_mix/1, t_without_in/1, t_ref/1, t_public_ref/1]). -export([t_require/1, t_query_enum/1, t_nullable/1, t_method/1, t_api_spec/1]). @@ -562,6 +562,8 @@ schema("/method/ok") -> schema("/method/error") -> #{operationId => test, bar => #{200 => <<"ok">>}}. +namespace() -> undefined. + fields(page) -> [ {per_page, diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index 5987ad8fa..85cc4b16b 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -680,6 +680,8 @@ to_schema(Object) -> post => #{responses => #{200 => Object, 201 => Object}} }. +namespace() -> undefined. + fields(good_ref) -> [ {'webhook-host', mk(emqx_schema:ip_port(), #{default => <<"127.0.0.1:80">>})}, diff --git a/apps/emqx_durable_storage/test/props/emqx_ds_message_storage_bitmask_shim.erl b/apps/emqx_durable_storage/test/props/emqx_ds_message_storage_bitmask_shim.erl index 9b5af9428..ddc199a44 100644 --- a/apps/emqx_durable_storage/test/props/emqx_ds_message_storage_bitmask_shim.erl +++ b/apps/emqx_durable_storage/test/props/emqx_ds_message_storage_bitmask_shim.erl @@ -11,9 +11,6 @@ -export([store/2]). -export([iterate/2]). --type topic() :: list(binary()). --type time() :: integer(). - -opaque t() :: ets:tid(). -export_type([t/0]). diff --git a/apps/emqx_durable_storage/test/props/payload_gen.erl b/apps/emqx_durable_storage/test/props/payload_gen.erl index 17e68f8d5..b969c0043 100644 --- a/apps/emqx_durable_storage/test/props/payload_gen.erl +++ b/apps/emqx_durable_storage/test/props/payload_gen.erl @@ -61,7 +61,7 @@ }. %% For performance reasons we treat regular lists as streams, see `next/1' --opaque cont(Data) :: +-type cont(Data) :: fun(() -> stream(Data)) | stream(Data). @@ -78,11 +78,11 @@ chunk_size :: non_neg_integer() }). --opaque chunk_state() :: #chunk_state{}. +-type chunk_state() :: #chunk_state{}. -record(interleave_state, {streams :: [{Tag :: term(), Stream :: term()}]}). --opaque interleave_state() :: #interleave_state{}. +-type interleave_state() :: #interleave_state{}. %% ============================================================================= %% API functions diff --git a/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl b/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl index 396a7fb37..548a5b531 100644 --- a/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl @@ -281,7 +281,7 @@ t_uuid_restored_from_file(Config) -> ), ok. -t_uuid_saved_to_file(Config) -> +t_uuid_saved_to_file(_Config) -> DataDir = emqx:data_dir(), NodeUUIDFile = filename:join(DataDir, "node.uuid"), ClusterUUIDFile = filename:join(DataDir, "cluster.uuid"), From 6143b3a2be6acc99ee3fd37834f19506cd5889fb Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 11 Dec 2023 14:08:45 +0100 Subject: [PATCH 23/50] fix(emqx_bridge_mongodb): use correct config when calling downgrade_type Also limit result in mongodb ..._to_bridge_v1_config to not include more fields than what's in the schema (most notably `description`). --- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- apps/emqx_bridge/src/emqx_bridge_lib.erl | 3 ++- .../src/emqx_bridge_mongodb_action_info.erl | 18 ++++++++++-------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 2343e3ccf..9ec0d440c 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -913,7 +913,7 @@ format_resource( redact( maps:merge( RawConfFull#{ - type => downgrade_type(Type, RawConf), + type => downgrade_type(Type, emqx_bridge_lib:get_conf(Type, BridgeName)), name => maps:get(<<"name">>, RawConf, BridgeName), node => Node }, diff --git a/apps/emqx_bridge/src/emqx_bridge_lib.erl b/apps/emqx_bridge/src/emqx_bridge_lib.erl index ed8e918fa..9386a38d3 100644 --- a/apps/emqx_bridge/src/emqx_bridge_lib.erl +++ b/apps/emqx_bridge/src/emqx_bridge_lib.erl @@ -18,7 +18,8 @@ -export([ maybe_withdraw_rule_action/3, upgrade_type/1, - downgrade_type/2 + downgrade_type/2, + get_conf/2 ]). %% @doc A bridge can be used as a rule action. diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl index 9fa19add2..be60959e8 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl @@ -26,19 +26,21 @@ -define(SCHEMA_MODULE, emqx_bridge_mongodb). -connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) -> - fix_v1_type( - maps:merge( +connector_action_config_to_bridge_v1_config( + #{<<"parameters">> := #{<<"mongo_type">> := MongoType}} = ConnectorConfig, + ActionConfig +) -> + MergedConfig = + emqx_utils_maps:deep_merge( maps:without( [<<"connector">>], emqx_utils_maps:unindent(<<"parameters">>, ActionConfig) ), emqx_utils_maps:unindent(<<"parameters">>, ConnectorConfig) - ) - ). - -fix_v1_type(#{<<"mongo_type">> := MongoType} = Conf) -> - Conf#{<<"type">> => v1_type(MongoType)}. + ), + BridgeV1Type = v1_type(MongoType), + BridgeV1Keys = schema_keys(BridgeV1Type), + maps:with(BridgeV1Keys, MergedConfig). bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) -> ActionTopLevelKeys = schema_keys(mongodb_action), From be31486983ce2c292a1087c4a8168fc9c9a08f56 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 11 Dec 2023 14:40:24 +0100 Subject: [PATCH 24/50] fix(emqx_utils): use deep_merge in unindent --- apps/emqx_utils/src/emqx_utils_maps.erl | 2 +- apps/emqx_utils/test/emqx_utils_maps_tests.erl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx_utils/src/emqx_utils_maps.erl b/apps/emqx_utils/src/emqx_utils_maps.erl index 1d97a926a..47b21a90a 100644 --- a/apps/emqx_utils/src/emqx_utils_maps.erl +++ b/apps/emqx_utils/src/emqx_utils_maps.erl @@ -345,7 +345,7 @@ indent(IndentKey, PickKeys, Map) -> -spec unindent(term(), map()) -> map(). unindent(Key, Map) -> - maps:merge( + deep_merge( maps:remove(Key, Map), maps:get(Key, Map, #{}) ). diff --git a/apps/emqx_utils/test/emqx_utils_maps_tests.erl b/apps/emqx_utils/test/emqx_utils_maps_tests.erl index 2778b5257..8b6e29235 100644 --- a/apps/emqx_utils/test/emqx_utils_maps_tests.erl +++ b/apps/emqx_utils/test/emqx_utils_maps_tests.erl @@ -170,5 +170,9 @@ map_indent_unindent_test_() -> ?_assertEqual( #{a => 1, b => 2}, unindent(c, M) + ), + ?_assertEqual( + #{a => #{c => 3, d => 4}}, + unindent(b, #{a => #{c => 3}, b => #{a => #{d => 4}}}) ) ]. From 31a7301f6f133e61fcb2fe5163436317bbf4d706 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 11 Dec 2023 14:50:38 +0100 Subject: [PATCH 25/50] refactor(emqx_bridge): be more generic in to_bridge_v1_config --- apps/emqx_bridge/src/emqx_action_info.erl | 19 +++++++- apps/emqx_bridge/src/emqx_bridge_v2.erl | 45 +++++++------------ .../src/emqx_bridge_mongodb_action_info.erl | 17 ------- 3 files changed, 32 insertions(+), 49 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index 9376eeef3..7d82f76d2 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -26,6 +26,7 @@ bridge_v1_type_to_action_type/1, is_action_type/1, registered_schema_modules/0, + connector_action_config_to_bridge_v1_config/2, connector_action_config_to_bridge_v1_config/3, has_custom_connector_action_config_to_bridge_v1_config/1, bridge_v1_config_to_connector_config/2, @@ -168,8 +169,22 @@ has_custom_connector_action_config_to_bridge_v1_config(ActionOrBridgeType) -> connector_action_config_to_bridge_v1_config(ActionOrBridgeType, ConnectorConfig, ActionConfig) -> Module = get_action_info_module(ActionOrBridgeType), - %% should only be called if defined - Module:connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig). + case erlang:function_exported(Module, connector_action_config_to_bridge_v1_config, 2) of + true -> + Module:connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig); + false -> + connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) + end. + +connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) -> + Merged = emqx_utils_maps:deep_merge( + maps:without( + [<<"connector">>], + emqx_utils_maps:unindent(<<"parameters">>, ActionConfig) + ), + emqx_utils_maps:unindent(<<"parameters">>, ConnectorConfig) + ), + maps:without([<<"description">>], Merged). has_custom_bridge_v1_config_to_connector_config(ActionOrBridgeType) -> Module = get_action_info_module(ActionOrBridgeType), diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index a9aeb0f8e..00d3ef6ec 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -1112,40 +1112,25 @@ bridge_v1_lookup_and_transform(ActionType, Name) -> not_bridge_v1_compatible_error() -> {error, not_bridge_v1_compatible}. +connector_raw_config(Connector, ConnectorType) -> + get_raw_with_defaults(Connector, ConnectorType, <<"connectors">>, emqx_connector_schema). + +action_raw_config(Action, ActionType) -> + get_raw_with_defaults(Action, ActionType, <<"actions">>, emqx_bridge_v2_schema). + +get_raw_with_defaults(Config, Type, TopLevelConf, SchemaModule) -> + RawConfig = maps:get(raw_config, Config), + fill_defaults(Type, RawConfig, TopLevelConf, SchemaModule). + bridge_v1_lookup_and_transform_helper( BridgeV1Type, BridgeName, ActionType, Action, ConnectorType, Connector ) -> - ConnectorRawConfig1 = maps:get(raw_config, Connector), - ConnectorRawConfig2 = fill_defaults( - ConnectorType, - ConnectorRawConfig1, - <<"connectors">>, - emqx_connector_schema + ConnectorRawConfig = connector_raw_config(Connector, ConnectorType), + ActionRawConfig = action_raw_config(Action, ActionType), + BridgeV1Config = emqx_action_info:connector_action_config_to_bridge_v1_config( + BridgeV1Type, ConnectorRawConfig, ActionRawConfig ), - ActionRawConfig1 = maps:get(raw_config, Action), - ActionRawConfig2 = fill_defaults( - ActionType, - ActionRawConfig1, - <<"actions">>, - emqx_bridge_v2_schema - ), - BridgeV1ConfigFinal = - case - emqx_action_info:has_custom_connector_action_config_to_bridge_v1_config(BridgeV1Type) - of - false -> - BridgeV1Config1 = maps:remove(<<"connector">>, ActionRawConfig2), - %% Move parameters to the top level - ParametersMap = maps:get(<<"parameters">>, BridgeV1Config1, #{}), - BridgeV1Config2 = maps:remove(<<"parameters">>, BridgeV1Config1), - BridgeV1Config3 = emqx_utils_maps:deep_merge(BridgeV1Config2, ParametersMap), - emqx_utils_maps:deep_merge(ConnectorRawConfig2, BridgeV1Config3); - true -> - emqx_action_info:connector_action_config_to_bridge_v1_config( - BridgeV1Type, ConnectorRawConfig2, ActionRawConfig2 - ) - end, - BridgeV1Tmp = maps:put(raw_config, BridgeV1ConfigFinal, Action), + BridgeV1Tmp = maps:put(raw_config, BridgeV1Config, Action), BridgeV1 = maps:remove(status, BridgeV1Tmp), BridgeV2Status = maps:get(status, Action, undefined), BridgeV2Error = maps:get(error, Action, undefined), diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl index be60959e8..060e6a17a 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl @@ -10,7 +10,6 @@ -export([ bridge_v1_config_to_action_config/2, bridge_v1_config_to_connector_config/1, - connector_action_config_to_bridge_v1_config/2, action_type_name/0, bridge_v1_type_name/0, connector_type_name/0, @@ -26,22 +25,6 @@ -define(SCHEMA_MODULE, emqx_bridge_mongodb). -connector_action_config_to_bridge_v1_config( - #{<<"parameters">> := #{<<"mongo_type">> := MongoType}} = ConnectorConfig, - ActionConfig -) -> - MergedConfig = - emqx_utils_maps:deep_merge( - maps:without( - [<<"connector">>], - emqx_utils_maps:unindent(<<"parameters">>, ActionConfig) - ), - emqx_utils_maps:unindent(<<"parameters">>, ConnectorConfig) - ), - BridgeV1Type = v1_type(MongoType), - BridgeV1Keys = schema_keys(BridgeV1Type), - maps:with(BridgeV1Keys, MergedConfig). - bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) -> ActionTopLevelKeys = schema_keys(mongodb_action), ActionParametersKeys = schema_keys(action_parameters), From 8e9cb6a928e97c457c7ee1339da8dea11869f11d Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 11 Dec 2023 14:54:10 +0100 Subject: [PATCH 26/50] style(emqx_bridge_mysql): minor cleanup --- apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl | 2 +- apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl index 05c782b96..6f1036600 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl @@ -129,7 +129,7 @@ fields(action) -> {mysql, mk( hoconsc:map(name, ref(?MODULE, mysql_action)), - #{desc => <<"MySQL Action Config">>, required => false} + #{desc => ?DESC("mysql_action"), required => false} )}; fields(mysql_action) -> emqx_bridge_v2_schema:make_producer_action_schema( diff --git a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl index f1b3f8260..65413817d 100644 --- a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl +++ b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl @@ -773,7 +773,6 @@ t_missing_table(Config) -> ), Val = integer_to_binary(erlang:unique_integer()), SentData = #{payload => Val, timestamp => 1668602148000}, - %Timeout = 1000, ResourceID = emqx_bridge_v2:id(BridgeType, Name), Request = {ResourceID, SentData}, Result = From a7c344ba292eb77323a459e7a4ec7071e86d26cf Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 11 Dec 2023 14:58:09 +0100 Subject: [PATCH 27/50] fix(emqx_bridge): remove unused --- apps/emqx_bridge/src/emqx_action_info.erl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index 7d82f76d2..c6304d9f7 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -28,7 +28,6 @@ registered_schema_modules/0, connector_action_config_to_bridge_v1_config/2, connector_action_config_to_bridge_v1_config/3, - has_custom_connector_action_config_to_bridge_v1_config/1, bridge_v1_config_to_connector_config/2, has_custom_bridge_v1_config_to_connector_config/1, bridge_v1_config_to_action_config/3, @@ -163,10 +162,6 @@ registered_schema_modules() -> Schemas = maps:get(action_type_to_schema_module, InfoMap), maps:to_list(Schemas). -has_custom_connector_action_config_to_bridge_v1_config(ActionOrBridgeType) -> - Module = get_action_info_module(ActionOrBridgeType), - erlang:function_exported(Module, connector_action_config_to_bridge_v1_config, 2). - connector_action_config_to_bridge_v1_config(ActionOrBridgeType, ConnectorConfig, ActionConfig) -> Module = get_action_info_module(ActionOrBridgeType), case erlang:function_exported(Module, connector_action_config_to_bridge_v1_config, 2) of From 2495f59c9174e4a458d8581224f90cb8e20498a1 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 11 Dec 2023 12:06:38 -0300 Subject: [PATCH 28/50] fix(actions): increment rule statistics even if channel is not installed Fixes new bug posted after https://emqx.atlassian.net/browse/EMQX-11494 was already fixed. Also reduces the usage of error throwing for flow control a bit. --- .../emqx_bridge/test/emqx_bridge_v2_SUITE.erl | 94 ++++++++++++++++++- .../src/emqx_resource_buffer_worker.erl | 60 +++++++----- 2 files changed, 129 insertions(+), 25 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl index 791997fc3..ce461427b 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -import(emqx_common_test_helpers, [on_exit/1]). @@ -343,7 +344,7 @@ t_send_message_through_rule(_) -> BridgeName = my_test_bridge, {ok, _} = emqx_bridge_v2:create(bridge_type(), BridgeName, bridge_config()), %% Create a rule to send message to the bridge - {ok, _} = emqx_rule_engine:create_rule( + {ok, #{id := RuleId}} = emqx_rule_engine:create_rule( #{ sql => <<"select * from \"t/a\"">>, id => atom_to_binary(?FUNCTION_NAME), @@ -357,6 +358,7 @@ t_send_message_through_rule(_) -> description => <<"bridge_v2 test rule">> } ), + on_exit(fun() -> emqx_rule_engine:delete_rule(RuleId) end), %% Register name for this process register(registered_process_name(), self()), %% Send message to the topic @@ -371,7 +373,6 @@ t_send_message_through_rule(_) -> ct:fail("Failed to receive message") end, unregister(registered_process_name()), - ok = emqx_rule_engine:delete_rule(atom_to_binary(?FUNCTION_NAME)), ok = emqx_bridge_v2:remove(bridge_type(), BridgeName), ok. @@ -894,6 +895,95 @@ t_lookup_status_when_connecting(_Config) -> ?assertMatch(#{status := ?status_disconnected}, ChannelData), ok. +t_rule_pointing_to_non_operational_channel(_Config) -> + %% Check that, if a rule sends a message to an action that is not yet installed and + %% uses `simple_async_internal_buffer', then it eventually increments the rule's + %% failed counter. + ResponseETS = ets:new(response_ets, [public]), + ets:insert(ResponseETS, {on_get_status_value, ?status_connecting}), + OnGetStatusFun = wrap_fun(fun() -> + ets:lookup_element(ResponseETS, on_get_status_value, 2) + end), + + ConnectorConfig = emqx_utils_maps:deep_merge(con_config(), #{ + <<"on_get_status_fun">> => OnGetStatusFun, + <<"resource_opts">> => #{<<"start_timeout">> => 100} + }), + ConnectorName = ?FUNCTION_NAME, + ct:pal("connector config:\n ~p", [ConnectorConfig]), + ?check_trace( + begin + %% FIXME: this should only matter for the action. yet, currently the query + %% mode from the connector is stored once by the resource manager and later + %% used to decide how to call the resource... + meck:new(con_mod(), [passthrough, no_history, non_strict]), + on_exit(fun() -> catch meck:unload([con_mod()]) end), + meck:expect(con_mod(), query_mode, 1, simple_async_internal_buffer), + meck:expect(con_mod(), callback_mode, 0, async_if_possible), + + {ok, _} = emqx_connector:create(con_type(), ConnectorName, ConnectorConfig), + + ActionName = my_test_action, + ChanStatusFun = wrap_fun(fun() -> ?status_disconnected end), + ActionConfig = (bridge_config())#{ + <<"on_get_channel_status_fun">> => ChanStatusFun, + <<"connector">> => atom_to_binary(ConnectorName) + }, + ct:pal("action config:\n ~p", [ActionConfig]), + {ok, _} = emqx_bridge_v2:create(bridge_type(), ActionName, ActionConfig), + + ?assertMatch( + {ok, #{ + error := <<"Not installed">>, + status := ?status_connecting, + resource_data := #{status := ?status_connecting} + }}, + emqx_bridge_v2:lookup(bridge_type(), ActionName) + ), + + {ok, #{id := RuleId}} = emqx_rule_engine:create_rule( + #{ + sql => <<"select * from \"t/a\"">>, + id => atom_to_binary(?FUNCTION_NAME), + actions => [ + << + (atom_to_binary(bridge_type()))/binary, + ":", + (atom_to_binary(ActionName))/binary + >> + ] + } + ), + on_exit(fun() -> emqx_rule_engine:delete_rule(RuleId) end), + + Msg = emqx_message:make(<<"t/a">>, <<"payload">>), + emqx:publish(Msg), + + ActionId = emqx_bridge_v2:id(bridge_type(), ActionName, ConnectorName), + ?assertEqual(1, emqx_resource_metrics:matched_get(ActionId)), + ?assertEqual(1, emqx_resource_metrics:failed_get(ActionId)), + ?retry( + _Sleep0 = 100, + _Attempts = 20, + ?assertMatch( + #{ + counters := + #{ + matched := 1, + 'actions.failed' := 1 + } + }, + emqx_metrics_worker:get_metrics(rule_metrics, RuleId) + ) + ), + + ok + end, + [] + ), + + ok. + %% Helper Functions wait_until(Fun) -> diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 98e1a785e..9c439206a 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -1122,14 +1122,14 @@ pre_query_channel_check({Id, _} = _Request, Channels, QueryOpts) when true -> ok; false -> - maybe_throw_channel_not_installed(Id, QueryOpts) + error_if_channel_is_not_installed(Id, QueryOpts) end; pre_query_channel_check({Id, _} = _Request, _Channels, QueryOpts) -> - maybe_throw_channel_not_installed(Id, QueryOpts); + error_if_channel_is_not_installed(Id, QueryOpts); pre_query_channel_check(_Request, _Channels, _QueryOpts) -> ok. -maybe_throw_channel_not_installed(Id, QueryOpts) -> +error_if_channel_is_not_installed(Id, QueryOpts) -> %% Fail with a recoverable error if the channel is not installed and there are buffer %% workers involved so that the operation can be retried. Otherwise, this is %% unrecoverable. It is emqx_resource_manager's responsibility to ensure that the @@ -1137,15 +1137,13 @@ maybe_throw_channel_not_installed(Id, QueryOpts) -> IsSimpleQuery = maps:get(simple_query, QueryOpts, false), case is_channel_id(Id) of true when IsSimpleQuery -> - error( + {error, {unrecoverable_error, - iolist_to_binary(io_lib:format("channel: \"~s\" not operational", [Id]))} - ); + iolist_to_binary(io_lib:format("channel: \"~s\" not operational", [Id]))}}; true -> - error( + {error, {recoverable_error, - iolist_to_binary(io_lib:format("channel: \"~s\" not operational", [Id]))} - ); + iolist_to_binary(io_lib:format("channel: \"~s\" not operational", [Id]))}}; false -> ok end. @@ -1201,8 +1199,12 @@ apply_query_fun( ?APPLY_RESOURCE( call_query, begin - pre_query_channel_check(Request, Channels, QueryOpts), - Mod:on_query(extract_connector_id(Id), Request, ResSt) + case pre_query_channel_check(Request, Channels, QueryOpts) of + ok -> + Mod:on_query(extract_connector_id(Id), Request, ResSt); + Error -> + Error + end end, Request ), @@ -1232,11 +1234,15 @@ apply_query_fun( AsyncWorkerMRef = undefined, InflightItem = ?INFLIGHT_ITEM(Ref, Query, IsRetriable, AsyncWorkerMRef), ok = inflight_append(InflightTID, InflightItem), - pre_query_channel_check(Request, Channels, QueryOpts), - Result = Mod:on_query_async( - extract_connector_id(Id), Request, {ReplyFun, [ReplyContext]}, ResSt - ), - {async_return, Result} + case pre_query_channel_check(Request, Channels, QueryOpts) of + ok -> + Result = Mod:on_query_async( + extract_connector_id(Id), Request, {ReplyFun, [ReplyContext]}, ResSt + ), + {async_return, Result}; + Error -> + maybe_reply_to(Error, QueryOpts) + end end, Request ); @@ -1259,8 +1265,12 @@ apply_query_fun( ?APPLY_RESOURCE( call_batch_query, begin - pre_query_channel_check(FirstRequest, Channels, QueryOpts), - Mod:on_batch_query(extract_connector_id(Id), Requests, ResSt) + case pre_query_channel_check(FirstRequest, Channels, QueryOpts) of + ok -> + Mod:on_batch_query(extract_connector_id(Id), Requests, ResSt); + Error -> + Error + end end, Batch ), @@ -1301,11 +1311,15 @@ apply_query_fun( AsyncWorkerMRef = undefined, InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, AsyncWorkerMRef), ok = inflight_append(InflightTID, InflightItem), - pre_query_channel_check(FirstRequest, Channels, QueryOpts), - Result = Mod:on_batch_query_async( - extract_connector_id(Id), Requests, {ReplyFun, [ReplyContext]}, ResSt - ), - {async_return, Result} + case pre_query_channel_check(FirstRequest, Channels, QueryOpts) of + ok -> + Result = Mod:on_batch_query_async( + extract_connector_id(Id), Requests, {ReplyFun, [ReplyContext]}, ResSt + ), + {async_return, Result}; + Error -> + maybe_reply_to(Error, QueryOpts) + end end, Batch ). From b4a5c141add5646a9e2d8f0a3df2001e761672c7 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 11 Dec 2023 14:06:01 -0300 Subject: [PATCH 29/50] fix(actions): use action query mode instead of connector's query mode --- .../emqx_bridge/test/emqx_bridge_v2_SUITE.erl | 80 +++++++++++++++++-- .../src/emqx_resource_buffer_worker.erl | 8 ++ 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl index ce461427b..bb0334bea 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl @@ -913,14 +913,6 @@ t_rule_pointing_to_non_operational_channel(_Config) -> ct:pal("connector config:\n ~p", [ConnectorConfig]), ?check_trace( begin - %% FIXME: this should only matter for the action. yet, currently the query - %% mode from the connector is stored once by the resource manager and later - %% used to decide how to call the resource... - meck:new(con_mod(), [passthrough, no_history, non_strict]), - on_exit(fun() -> catch meck:unload([con_mod()]) end), - meck:expect(con_mod(), query_mode, 1, simple_async_internal_buffer), - meck:expect(con_mod(), callback_mode, 0, async_if_possible), - {ok, _} = emqx_connector:create(con_type(), ConnectorName, ConnectorConfig), ActionName = my_test_action, @@ -930,6 +922,12 @@ t_rule_pointing_to_non_operational_channel(_Config) -> <<"connector">> => atom_to_binary(ConnectorName) }, ct:pal("action config:\n ~p", [ActionConfig]), + + meck:new(con_mod(), [passthrough, no_history, non_strict]), + on_exit(fun() -> catch meck:unload([con_mod()]) end), + meck:expect(con_mod(), query_mode, 1, simple_async_internal_buffer), + meck:expect(con_mod(), callback_mode, 0, async_if_possible), + {ok, _} = emqx_bridge_v2:create(bridge_type(), ActionName, ActionConfig), ?assertMatch( @@ -984,6 +982,72 @@ t_rule_pointing_to_non_operational_channel(_Config) -> ok. +t_query_uses_action_query_mode(_Config) -> + %% Check that we compute the query mode from the action and not from the connector + %% when querying the resource. + + %% Set one query mode for the connector... + meck:new(con_mod(), [passthrough, no_history, non_strict]), + on_exit(fun() -> catch meck:unload([con_mod()]) end), + meck:expect(con_mod(), query_mode, 1, sync), + meck:expect(con_mod(), callback_mode, 0, always_sync), + + ConnectorConfig = emqx_utils_maps:deep_merge(con_config(), #{ + <<"resource_opts">> => #{<<"start_timeout">> => 100} + }), + ConnectorName = ?FUNCTION_NAME, + ct:pal("connector config:\n ~p", [ConnectorConfig]), + ?check_trace( + begin + {ok, _} = emqx_connector:create(con_type(), ConnectorName, ConnectorConfig), + + ActionName = my_test_action, + ActionConfig = (bridge_config())#{ + <<"connector">> => atom_to_binary(ConnectorName) + }, + ct:pal("action config:\n ~p", [ActionConfig]), + + %% ... now we use a quite different query mode for the action + meck:expect(con_mod(), query_mode, 1, simple_async_internal_buffer), + meck:expect(con_mod(), callback_mode, 0, async_if_possible), + + {ok, _} = emqx_bridge_v2:create(bridge_type(), ActionName, ActionConfig), + + {ok, #{id := RuleId}} = emqx_rule_engine:create_rule( + #{ + sql => <<"select * from \"t/a\"">>, + id => atom_to_binary(?FUNCTION_NAME), + actions => [ + << + (atom_to_binary(bridge_type()))/binary, + ":", + (atom_to_binary(ActionName))/binary + >> + ] + } + ), + on_exit(fun() -> emqx_rule_engine:delete_rule(RuleId) end), + + Msg = emqx_message:make(<<"t/a">>, <<"payload">>), + {_, {ok, _}} = + ?wait_async_action( + emqx:publish(Msg), + #{?snk_kind := call_query}, + 2_000 + ), + + ok + end, + fun(Trace) -> + ?assertMatch( + [#{query_mode := simple_async_internal_buffer}], + ?of_kind(simple_query_override, Trace) + ), + ok + end + ), + ok. + %% Helper Functions wait_until(Fun) -> diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 9c439206a..0d3b9cf97 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -1148,6 +1148,14 @@ error_if_channel_is_not_installed(Id, QueryOpts) -> ok end. +do_call_query(QM, Id, Index, Ref, Query, #{query_mode := ReqQM} = QueryOpts, Resource) when + ReqQM =:= simple_sync_internal_buffer; ReqQM =:= simple_async_internal_buffer +-> + %% The query overrides the query mode of the resource, send even in disconnected state + ?tp(simple_query_override, #{query_mode => ReqQM}), + #{mod := Mod, state := ResSt, callback_mode := CBM, added_channels := Channels} = Resource, + CallMode = call_mode(QM, CBM), + apply_query_fun(CallMode, Mod, Id, Index, Ref, Query, ResSt, Channels, QueryOpts); do_call_query(QM, Id, Index, Ref, Query, QueryOpts, #{query_mode := ResQM} = Resource) when ResQM =:= simple_sync_internal_buffer; ResQM =:= simple_async_internal_buffer -> From fbbb55633d56ea2a5ccbe1547ea0d773cb4ef895 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 11 Dec 2023 14:13:40 -0300 Subject: [PATCH 30/50] fix(connector_schema): remove `query_mode` from `resource_opts` The connector query mode is inferred during its creation, and later it must be overridden by an action, anyway. --- apps/emqx_bridge/test/emqx_bridge_v2_tests.erl | 1 - apps/emqx_connector/src/schema/emqx_connector_schema.erl | 1 - 2 files changed, 2 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl b/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl index 9fe2d2e76..e90100995 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl @@ -69,7 +69,6 @@ connector_resource_opts_test() -> %% These are used by `emqx_resource_manager' itself to manage the resource lifecycle. MinimumROFields = [ health_check_interval, - query_mode, start_after_created, start_timeout ], diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 6515a45d8..7b47c06d9 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -533,7 +533,6 @@ resource_opts_ref(Module, RefName) -> common_resource_opts_subfields() -> [ health_check_interval, - query_mode, start_after_created, start_timeout ]. From f7adaa891175a1f7bf5c7504c7aaa3dc58a21fe5 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 11 Dec 2023 15:03:20 -0300 Subject: [PATCH 31/50] fix(postgres_schema): fix usages of `resource_opts` in connector and action schemas Fixes https://emqx.atlassian.net/browse/EMQX-11569 --- apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl | 2 +- .../src/schema/emqx_postgresql_connector_schema.erl | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index b271f4259..3a2be82f8 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -177,7 +177,7 @@ roots() -> fields(actions) -> registered_schema_fields(); fields(resource_opts) -> - emqx_resource_schema:create_opts(_Overrides = []). + resource_opts_fields(_Overrides = []). registered_schema_fields() -> [ diff --git a/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl b/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl index 304a7356c..b4e04e982 100644 --- a/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl +++ b/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl @@ -85,7 +85,10 @@ fields({Field, Type}) when Field == "put_connector"; Field == "post_connector" -> - emqx_connector_schema:api_fields(Field, Type, fields("connection_fields")). + Fields = + fields("connection_fields") ++ + emqx_connector_schema:resource_opts_ref(?MODULE, resource_opts), + emqx_connector_schema:api_fields(Field, Type, Fields). server() -> Meta = #{desc => ?DESC("server")}, From 593283df93a5d2cd251eaa57f39891e909e456bb Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 11 Dec 2023 14:43:20 -0300 Subject: [PATCH 32/50] fix(redis_bridge_schema): use correct `resource_opts` subfields for connector and action --- .../src/schema/emqx_bridge_v2_schema.erl | 19 +++++++++++---- .../src/emqx_bridge_redis.erl | 5 ++-- .../src/emqx_bridge_redis_action_info.erl | 14 +++++++++-- .../src/emqx_bridge_redis_schema.erl | 22 ++++++++---------- .../test/emqx_bridge_redis_SUITE.erl | 23 ++++++++++++------- .../src/schema/emqx_connector_schema.erl | 16 ++++++++----- 6 files changed, 65 insertions(+), 34 deletions(-) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index b271f4259..f052184ef 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -48,7 +48,8 @@ -export([ make_producer_action_schema/1, make_consumer_action_schema/1, - top_level_common_action_keys/0 + top_level_common_action_keys/0, + project_to_actions_resource_opts/1 ]). -export_type([action_type/0]). @@ -203,8 +204,8 @@ types_sc() -> resource_opts_fields() -> resource_opts_fields(_Overrides = []). -resource_opts_fields(Overrides) -> - ActionROFields = [ +common_resource_opts_subfields() -> + [ batch_size, batch_time, buffer_mode, @@ -219,7 +220,13 @@ resource_opts_fields(Overrides) -> start_after_created, start_timeout, worker_pool_size - ], + ]. + +common_resource_opts_subfields_bin() -> + lists:map(fun atom_to_binary/1, common_resource_opts_subfields()). + +resource_opts_fields(Overrides) -> + ActionROFields = common_resource_opts_subfields(), lists:filter( fun({Key, _Sc}) -> lists:member(Key, ActionROFields) end, emqx_resource_schema:create_opts(Overrides) @@ -274,6 +281,10 @@ make_consumer_action_schema(ActionParametersRef) -> })} ]. +project_to_actions_resource_opts(OldResourceOpts) -> + Subfields = common_resource_opts_subfields_bin(), + maps:with(Subfields, OldResourceOpts). + -ifdef(TEST). -include_lib("hocon/include/hocon_types.hrl"). schema_homogeneous_test() -> diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis.erl index beafc8775..5bab0cb32 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis.erl @@ -120,6 +120,7 @@ fields("get_sentinel") -> method_fields(get, redis_sentinel); fields("get_cluster") -> method_fields(get, redis_cluster); +%% old bridge v1 schema fields(Type) when Type == redis_single orelse Type == redis_sentinel orelse Type == redis_cluster -> @@ -147,7 +148,7 @@ redis_bridge_common_fields(Type) -> {local_topic, mk(binary(), #{required => false, desc => ?DESC("desc_local_topic")})} | fields(action_parameters) ] ++ - resource_fields(Type). + v1_resource_fields(Type). connector_fields(Type) -> emqx_redis:fields(Type). @@ -158,7 +159,7 @@ type_name_fields(Type) -> {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})} ]. -resource_fields(Type) -> +v1_resource_fields(Type) -> [ {resource_opts, mk( diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_action_info.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_action_info.erl index 22ed40093..6ead37170 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_action_info.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_action_info.erl @@ -43,7 +43,12 @@ bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) -> ActionTopLevelKeys = schema_keys(?SCHEMA_MODULE:fields(redis_action)), ActionParametersKeys = schema_keys(emqx_bridge_redis:fields(action_parameters)), ActionKeys = ActionTopLevelKeys ++ ActionParametersKeys, - ActionConfig = make_config_map(ActionKeys, ActionParametersKeys, BridgeV1Config), + ActionConfig0 = make_config_map(ActionKeys, ActionParametersKeys, BridgeV1Config), + ActionConfig = emqx_utils_maps:update_if_present( + <<"resource_opts">>, + fun emqx_bridge_v2_schema:project_to_actions_resource_opts/1, + ActionConfig0 + ), ActionConfig#{<<"connector">> => ConnectorName}. bridge_v1_config_to_connector_config(BridgeV1Config) -> @@ -57,7 +62,12 @@ bridge_v1_config_to_connector_config(BridgeV1Config) -> (maps:keys(BridgeV1Config) -- (ActionKeys -- ConnectorTopLevelKeys)) ++ [<<"redis_type">>], ConnectorParametersKeys = ConnectorKeys -- ConnectorTopLevelKeys, - make_config_map(ConnectorKeys, ConnectorParametersKeys, BridgeV1Config). + ConnectorConfig0 = make_config_map(ConnectorKeys, ConnectorParametersKeys, BridgeV1Config), + emqx_utils_maps:update_if_present( + <<"resource_opts">>, + fun emqx_connector_schema:project_to_connector_resource_opts/1, + ConnectorConfig0 + ). %%------------------------------------------------------------------------------------------ %% Internal helper fns diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl index b1a27d1ce..6a3f1005f 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl @@ -51,8 +51,10 @@ fields("config_connector") -> )} ] ++ emqx_redis:redis_fields() ++ - emqx_connector_schema:resource_opts_ref(?MODULE, resource_opts) ++ + emqx_connector_schema:resource_opts_ref(?MODULE, connector_resource_opts) ++ emqx_connector_schema_lib:ssl_fields(); +fields(connector_resource_opts) -> + emqx_connector_schema:resource_opts_fields(); fields(action) -> {?TYPE, ?HOCON( @@ -74,15 +76,7 @@ fields(redis_action) -> } ) ), - ResOpts = - {resource_opts, - ?HOCON( - ?R_REF(resource_opts), - #{ - required => true, - desc => ?DESC(emqx_resource_schema, resource_opts) - } - )}, + [ResOpts] = emqx_connector_schema:resource_opts_ref(?MODULE, action_resource_opts), RedisType = {redis_type, ?HOCON( @@ -90,8 +84,8 @@ fields(redis_action) -> #{required => true, desc => ?DESC(redis_type)} )}, [RedisType | lists:keyreplace(resource_opts, 1, Schema, ResOpts)]; -fields(resource_opts) -> - emqx_resource_schema:create_opts([ +fields(action_resource_opts) -> + emqx_bridge_v2_schema:resource_opts_fields([ {batch_size, #{desc => ?DESC(batch_size)}}, {batch_time, #{desc => ?DESC(batch_time)}} ]); @@ -124,6 +118,10 @@ desc(redis_action) -> ?DESC(redis_action); desc(resource_opts) -> ?DESC(emqx_resource_schema, resource_opts); +desc(connector_resource_opts) -> + ?DESC(emqx_resource_schema, "resource_opts"); +desc(action_resource_opts) -> + ?DESC(emqx_resource_schema, "resource_opts"); desc(_Name) -> undefined. diff --git a/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl b/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl index 125d84d0f..508051f93 100644 --- a/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl +++ b/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl @@ -123,10 +123,19 @@ wait_for_ci_redis(Checks, Config) -> ProxyHost = os:getenv("PROXY_HOST", ?PROXY_HOST), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", ?PROXY_PORT)), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - ok = emqx_common_test_helpers:start_apps([ - emqx_conf, emqx_resource, emqx_connector, emqx_bridge, emqx_rule_engine - ]), + Apps = emqx_cth_suite:start( + [ + emqx, + emqx_conf, + emqx_resource, + emqx_connector, + emqx_bridge, + emqx_rule_engine + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), [ + {apps, Apps}, {proxy_host, ProxyHost}, {proxy_port, ProxyPort} | Config @@ -143,11 +152,9 @@ redis_checks() -> 1 end. -end_per_suite(_Config) -> - ok = emqx_bridge_v2_SUITE:delete_all_bridges_and_connectors(), - ok = emqx_common_test_helpers:stop_apps([emqx_conf]), - ok = emqx_connector_test_helpers:stop_apps([emqx_rule_engine, emqx_bridge, emqx_resource]), - _ = application:stop(emqx_connector), +end_per_suite(Config) -> + Apps = ?config(apps, Config), + emqx_cth_suite:stop(Apps), ok. init_per_testcase(Testcase, Config0) -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 7b47c06d9..381fe2c82 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -28,7 +28,8 @@ -export([ transform_bridges_v1_to_connectors_and_bridges_v2/1, transform_bridge_v1_config_to_action_config/4, - top_level_common_connector_keys/0 + top_level_common_connector_keys/0, + project_to_connector_resource_opts/1 ]). -export([roots/0, fields/1, desc/1, namespace/0, tags/0]). @@ -195,7 +196,7 @@ split_bridge_to_connector_and_action( case maps:is_key(ConnectorFieldNameBin, BridgeV1Conf) of true -> PrevFieldConfig = - project_to_connector_resource_opts( + maybe_project_to_connector_resource_opts( ConnectorFieldNameBin, maps:get(ConnectorFieldNameBin, BridgeV1Conf) ), @@ -231,12 +232,15 @@ split_bridge_to_connector_and_action( end, {BridgeType, BridgeName, ActionMap, ConnectorName, ConnectorMap}. -project_to_connector_resource_opts(<<"resource_opts">>, OldResourceOpts) -> - Subfields = common_resource_opts_subfields_bin(), - maps:with(Subfields, OldResourceOpts); -project_to_connector_resource_opts(_, OldConfig) -> +maybe_project_to_connector_resource_opts(<<"resource_opts">>, OldResourceOpts) -> + project_to_connector_resource_opts(OldResourceOpts); +maybe_project_to_connector_resource_opts(_, OldConfig) -> OldConfig. +project_to_connector_resource_opts(OldResourceOpts) -> + Subfields = common_resource_opts_subfields_bin(), + maps:with(Subfields, OldResourceOpts). + transform_bridge_v1_config_to_action_config( BridgeV1Conf, ConnectorName, ConnectorConfSchemaMod, ConnectorConfSchemaName ) -> From 30719d286a85baeac87e662088dfa4ecedad9814 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 11 Dec 2023 14:32:16 -0300 Subject: [PATCH 33/50] fix(http_bridge_schema): use correct `resource_opts` subfields for connector and action --- .../src/emqx_bridge_http_action_info.erl | 14 ++++- .../src/emqx_bridge_http_schema.erl | 53 ++++++++++--------- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl index 457d8ff4b..f2b47c122 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl @@ -69,13 +69,23 @@ connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) -> bridge_v1_config_to_connector_config(BridgeV1Conf) -> %% To statisfy the emqx_bridge_api_SUITE:t_http_crud_apis/1 ok = validate_webhook_url(maps:get(<<"url">>, BridgeV1Conf, undefined)), - maps:without(?REMOVED_KEYS ++ ?ACTION_KEYS ++ ?PARAMETER_KEYS, BridgeV1Conf). + ConnectorConfig0 = maps:without(?REMOVED_KEYS ++ ?ACTION_KEYS ++ ?PARAMETER_KEYS, BridgeV1Conf), + emqx_utils_maps:update_if_present( + <<"resource_opts">>, + fun emqx_connector_schema:project_to_connector_resource_opts/1, + ConnectorConfig0 + ). bridge_v1_config_to_action_config(BridgeV1Conf, ConnectorName) -> Parameters = maps:with(?PARAMETER_KEYS, BridgeV1Conf), Parameters1 = Parameters#{<<"path">> => <<>>, <<"headers">> => #{}}, CommonKeys = [<<"enable">>, <<"description">>], - ActionConfig = maps:with(?ACTION_KEYS ++ CommonKeys, BridgeV1Conf), + ActionConfig0 = maps:with(?ACTION_KEYS ++ CommonKeys, BridgeV1Conf), + ActionConfig = emqx_utils_maps:update_if_present( + <<"resource_opts">>, + fun emqx_bridge_v2_schema:project_to_actions_resource_opts/1, + ActionConfig0 + ), ActionConfig#{<<"parameters">> => Parameters1, <<"connector">> => ConnectorName}. %%-------------------------------------------------------------------- diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl index 225405a4a..91ad25d31 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl @@ -48,7 +48,15 @@ fields("get") -> %%--- v1 bridges config file %% see: emqx_bridge_schema:fields(bridges) fields("config") -> - basic_config() ++ request_config(); + basic_config() ++ + request_config() ++ + emqx_connector_schema:resource_opts_ref(?MODULE, "v1_resource_opts"); +fields("v1_resource_opts") -> + UnsupportedOpts = [enable_batch, batch_size, batch_time], + lists:filter( + fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, + emqx_resource_schema:fields("creation_opts") + ); %%-------------------------------------------------------------------- %% v2: configuration fields(action) -> @@ -89,7 +97,13 @@ fields("http_action") -> required => true, desc => ?DESC("config_parameters_opts") })} - ] ++ http_resource_opts(); + ] ++ emqx_connector_schema:resource_opts_ref(?MODULE, action_resource_opts); +fields(action_resource_opts) -> + UnsupportedOpts = [batch_size, batch_time], + lists:filter( + fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, + emqx_bridge_v2_schema:resource_opts_fields() + ); fields("parameters_opts") -> [ {path, @@ -129,20 +143,20 @@ fields("config_connector") -> } )}, {description, emqx_schema:description_schema()} - ] ++ connector_url_headers() ++ connector_opts(); -%%-------------------------------------------------------------------- -%% v1/v2 -fields("resource_opts") -> - UnsupportedOpts = [enable_batch, batch_size, batch_time], - lists:filter( - fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, - emqx_resource_schema:fields("creation_opts") - ). + ] ++ connector_url_headers() ++ + connector_opts() ++ + emqx_connector_schema:resource_opts_ref(?MODULE, connector_resource_opts); +fields(connector_resource_opts) -> + emqx_connector_schema:resource_opts_fields(). desc("config") -> ?DESC("desc_config"); -desc("resource_opts") -> +desc("v1_resource_opts") -> ?DESC(emqx_resource_schema, "creation_opts"); +desc(connector_resource_opts) -> + ?DESC(emqx_resource_schema, "resource_opts"); +desc(action_resource_opts) -> + ?DESC(emqx_resource_schema, "resource_opts"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for WebHook using `", string:to_upper(Method), "` method."]; desc("config_connector") -> @@ -304,23 +318,10 @@ request_timeout_field() -> } )}. -http_resource_opts() -> - [ - {resource_opts, - mk( - ref(?MODULE, "resource_opts"), - #{ - required => false, - default => #{}, - desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) - } - )} - ]. - connector_opts() -> mark_request_field_deperecated( proplists:delete(max_retries, emqx_bridge_http_connector:fields(config)) - ) ++ http_resource_opts(). + ). mark_request_field_deperecated(Fields) -> lists:map( From b80c9b0863b8d30992fec0b4cc145bee7763d357 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 11 Dec 2023 14:46:17 -0300 Subject: [PATCH 34/50] fix(actions_schema): remove redundant `resource_opts` subfields for actions Buffer workers don't use those fields. --- apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index f052184ef..7670af52e 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -217,8 +217,6 @@ common_resource_opts_subfields() -> query_mode, request_ttl, resume_interval, - start_after_created, - start_timeout, worker_pool_size ]. From cd1365d7537cc4ddb2c120fb1b67fdd96c775bc4 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 12 Dec 2023 10:15:53 +0800 Subject: [PATCH 35/50] fix: don't require path in http action(fill default path) --- .../emqx_bridge_http/src/emqx_bridge_http_action_info.erl | 2 +- apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl index 457d8ff4b..4825cdbb3 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl @@ -67,7 +67,7 @@ connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) -> }. bridge_v1_config_to_connector_config(BridgeV1Conf) -> - %% To statisfy the emqx_bridge_api_SUITE:t_http_crud_apis/1 + %% To satisfy the emqx_bridge_api_SUITE:t_http_crud_apis/1 ok = validate_webhook_url(maps:get(<<"url">>, BridgeV1Conf, undefined)), maps:without(?REMOVED_KEYS ++ ?ACTION_KEYS ++ ?PARAMETER_KEYS, BridgeV1Conf). diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 5ecfa76d1..1777fbe1f 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -565,12 +565,8 @@ preprocess_request(undefined) -> undefined; preprocess_request(Req) when map_size(Req) == 0 -> undefined; -preprocess_request( - #{ - method := Method, - path := Path - } = Req -) -> +preprocess_request(#{method := Method} = Req) -> + Path = maps:get(path, Req, <<>>), Headers = maps:get(headers, Req, []), #{ method => parse_template(to_bin(Method)), From 363055a32eb6feb82e043f7f63e6da230dbc857e Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 12 Dec 2023 09:48:03 +0800 Subject: [PATCH 36/50] fix: add descriptions fields to bridge_v1 --- apps/emqx_bridge/src/schema/emqx_bridge_schema.erl | 5 ++++- apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 1c4d5365d..aa58d825e 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -124,7 +124,10 @@ common_bridge_fields() -> desc => ?DESC("desc_enable"), default => true } - )} + )}, + %% Create v2 connector then usr v1 /bridges_probe api to test connector + %% /bridges_probe should pass through v2 connector's description. + {description, emqx_schema:description_schema()} ]. status_fields() -> diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index d54330d54..0ac1b2e3c 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -1047,6 +1047,13 @@ t_bridges_probe(Config) -> ?HTTP_BRIDGE(URL), Config ), + %% with descriptions is ok. + {ok, 204, <<>>} = request( + post, + uri(["bridges_probe"]), + (?HTTP_BRIDGE(URL))#{<<"description">> => <<"Test Description">>}, + Config + ), ?assertMatch( {ok, 400, #{ From e2e7f96515c519e35f2341141f2b14d1c0e9bba8 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 12 Dec 2023 14:29:39 +0800 Subject: [PATCH 37/50] fix: incorrect behavior in audit logs during stop and start of EMQX --- apps/emqx_ctl/src/emqx_ctl.erl | 52 ++++++++++++------- .../src/emqx_machine_terminator.erl | 2 +- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/apps/emqx_ctl/src/emqx_ctl.erl b/apps/emqx_ctl/src/emqx_ctl.erl index 60413fb83..eaaf15574 100644 --- a/apps/emqx_ctl/src/emqx_ctl.erl +++ b/apps/emqx_ctl/src/emqx_ctl.erl @@ -336,30 +336,42 @@ audit_log(Level, From, Log) -> {error, _} -> ignore; {ok, {Mod, Fun}} -> - try - apply(Mod, Fun, [Level, From, normalize_audit_log_args(Log)]) - catch - _:{aborted, {no_exists, emqx_audit}} -> - case Log of - #{cmd := cluster, args := ["leave"]} -> - ok; - _ -> - ?LOG_ERROR(#{ - msg => "ctl_command_crashed", - reason => "emqx_audit table not found", - log => normalize_audit_log_args(Log), - from => From - }) - end; - _:Reason:Stacktrace -> + case prune_unnecessary_log(Log) of + false -> ok; + {ok, Log1} -> apply_audit_command(Log1, Mod, Fun, Level, From) + end + end. + +apply_audit_command(Log, Mod, Fun, Level, From) -> + try + apply(Mod, Fun, [Level, From, Log]) + catch + _:{aborted, {no_exists, emqx_audit}} -> + case Log of + #{cmd := cluster, args := [<<"leave">>]} -> + ok; + _ -> ?LOG_ERROR(#{ msg => "ctl_command_crashed", - stacktrace => Stacktrace, - reason => Reason, - log => normalize_audit_log_args(Log), + reason => "emqx_audit table not found", + log => Log, from => From }) - end + end; + _:Reason:Stacktrace -> + ?LOG_ERROR(#{ + msg => "ctl_command_crashed", + stacktrace => Stacktrace, + reason => Reason, + log => Log, + from => From + }) + end. + +prune_unnecessary_log(Log) -> + case normalize_audit_log_args(Log) of + #{args := [<<"emqx:is_running()">>]} -> false; + Log1 -> {ok, Log1} end. audit_level(ok, _Duration) -> info; diff --git a/apps/emqx_machine/src/emqx_machine_terminator.erl b/apps/emqx_machine/src/emqx_machine_terminator.erl index 28603b9f6..67a21c7fd 100644 --- a/apps/emqx_machine/src/emqx_machine_terminator.erl +++ b/apps/emqx_machine/src/emqx_machine_terminator.erl @@ -69,7 +69,7 @@ graceful() -> graceful_wait() -> ?AUDIT(alert, #{ cmd => emqx, - args => ["stop"], + args => [<<"stop">>], version => emqx_release:version(), from => cli, duration_ms => element(1, erlang:statistics(wall_clock)) From 1807df157cea022410e67c636647794b1a894f8f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 12 Dec 2023 17:27:31 +0800 Subject: [PATCH 38/50] fix: license watermark percent support float --- apps/emqx/rebar.config | 2 +- apps/emqx_license/test/emqx_license_http_api_SUITE.erl | 4 ++-- mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index bcc007141..b5cf22798 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -30,7 +30,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.9"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.17.0"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.0"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.1"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.2"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/apps/emqx_license/test/emqx_license_http_api_SUITE.erl b/apps/emqx_license/test/emqx_license_http_api_SUITE.erl index ad16d75c9..799e4f591 100644 --- a/apps/emqx_license/test/emqx_license_http_api_SUITE.erl +++ b/apps/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -205,14 +205,14 @@ t_license_setting(_Config) -> ?assertEqual(0.55, emqx_config:get([license, connection_high_watermark])), %% update - Low1 = <<"50%">>, + Low1 = <<"50.12%">>, High1 = <<"100%">>, UpdateRes1 = request(put, uri(["license", "setting"]), #{ <<"connection_low_watermark">> => Low1, <<"connection_high_watermark">> => High1 }), validate_setting(UpdateRes1, Low1, High1), - ?assertEqual(0.5, emqx_config:get([license, connection_low_watermark])), + ?assertEqual(0.5012, emqx_config:get([license, connection_low_watermark])), ?assertEqual(1.0, emqx_config:get([license, connection_high_watermark])), %% update bad setting low >= high diff --git a/mix.exs b/mix.exs index 74f4ae105..26f49b318 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.40.1", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.40.2", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.3", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 7380347f2..eaf82f47b 100644 --- a/rebar.config +++ b/rebar.config @@ -84,7 +84,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.1"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.2"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From ea6f5764bb4699a251c15afdf3f99aa622e949a9 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 12 Dec 2023 12:05:46 +0200 Subject: [PATCH 39/50] test: remove debug message --- apps/emqx_opentelemetry/test/emqx_otel_schema_SUITE.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/emqx_opentelemetry/test/emqx_otel_schema_SUITE.erl b/apps/emqx_opentelemetry/test/emqx_otel_schema_SUITE.erl index f5682dcad..abc6548f9 100644 --- a/apps/emqx_opentelemetry/test/emqx_otel_schema_SUITE.erl +++ b/apps/emqx_opentelemetry/test/emqx_otel_schema_SUITE.erl @@ -164,7 +164,6 @@ t_old_conf_disabled_exporter(_Config) -> ?assertNot(erlang:is_map_key(interval, maps:get(exporter, OtelConf))). t_old_conf_exporter(_Config) -> - io:format(user, "TC running: ~p~n", [?FUNCTION_NAME]), OtelConf = emqx:get_config([opentelemetry]), ?assertMatch( #{ From f529bda8b7f8f519cf56bed0914816d29533433d Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 12 Dec 2023 12:07:59 +0200 Subject: [PATCH 40/50] fix(opentelemetry_schema): decrease scheduled_delay/interval importance to medium These fields should be shown in API schema. --- apps/emqx_opentelemetry/src/emqx_otel_schema.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl index bcd0b8dcf..420bf23f2 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl @@ -108,7 +108,7 @@ fields("otel_metrics") -> aliases => [scheduled_delay], default => <<"10s">>, desc => ?DESC(scheduled_delay), - importance => ?IMPORTANCE_HIDDEN + importance => ?IMPORTANCE_MEDIUM } )} ]; @@ -156,7 +156,7 @@ fields("otel_logs") -> #{ default => <<"1s">>, desc => ?DESC(scheduled_delay), - importance => ?IMPORTANCE_HIDDEN + importance => ?IMPORTANCE_MEDIUM } )} ]; @@ -195,7 +195,7 @@ fields("otel_traces") -> #{ default => <<"5s">>, desc => ?DESC(scheduled_delay), - importance => ?IMPORTANCE_HIDDEN + importance => ?IMPORTANCE_MEDIUM } )}, {filter, From 36b12a01c96fe829e6cb05c756332184bfa23cb3 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 12 Dec 2023 13:06:20 +0200 Subject: [PATCH 41/50] chore: fix typos in comments --- apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl b/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl index 88917d7e3..f1554094f 100644 --- a/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl +++ b/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl @@ -29,7 +29,7 @@ %% How to run it locally: %% 1. Uncomment networks in .ci/docker-compose-file/docker-compose-otel.yaml, %% Uncomment OTLP gRPC ports mappings for otel-collector and otel-collector-tls services. -%% Uncomment jaeger-all-in-one prots maooing. +%% Uncomment jaeger-all-in-one ports mapping. %% 2. Start deps services: %% DOCKER_USER="$(id -u)" docker-compose -f .ci/docker-compose-file/docker-compose-otel.yaml up %% 3. Run tests with special env variables: From 3c3452c1dd2c702c174940e6f60cb41a0839ff4a Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 12 Dec 2023 13:07:08 +0200 Subject: [PATCH 42/50] fix(emqx_otel_schema): add `enable` field to SSL opts --- apps/emqx_opentelemetry/src/emqx_otel_config.erl | 2 +- apps/emqx_opentelemetry/src/emqx_otel_schema.erl | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/apps/emqx_opentelemetry/src/emqx_otel_config.erl b/apps/emqx_opentelemetry/src/emqx_otel_config.erl index 0d2f9988b..c16b3385a 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_config.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_config.erl @@ -150,7 +150,7 @@ tr_handler_conf(#{logs := LogsConf, exporter := ExporterConf}) -> ssl_opts(Endpoint, SSLOpts) -> case is_ssl(Endpoint) of true -> - emqx_tls_lib:to_client_opts(SSLOpts#{enable => true}); + emqx_tls_lib:to_client_opts(SSLOpts); false -> [] end. diff --git a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl index 420bf23f2..be14a2b29 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl @@ -230,16 +230,14 @@ fields("otel_exporter") -> )}, {ssl_options, ?HOCON( - ?R_REF("ssl_opts"), + ?R_REF(emqx_schema, "ssl_client_opts"), #{ desc => ?DESC(exporter_ssl), + default => #{<<"enable">> => false}, importance => ?IMPORTANCE_LOW } )} ]; -fields("ssl_opts") -> - Schema = emqx_schema:client_ssl_opts_schema(#{}), - lists:keydelete("enable", 1, Schema); fields("trace_filter") -> %% More filters can be implemented in future, e.g. topic, clientid [ @@ -259,6 +257,5 @@ desc("otel_exporter") -> ?DESC(otel_exporter); desc("otel_logs") -> ?DESC(otel_logs); desc("otel_metrics") -> ?DESC(otel_metrics); desc("otel_traces") -> ?DESC(otel_traces); -desc("ssl_opts") -> ?DESC(exporter_ssl); desc("trace_filter") -> ?DESC(trace_filter); desc(_) -> undefined. From 6e49cce0556a54e78662c16a1e754e0ea8d739a3 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 12 Dec 2023 11:20:33 -0300 Subject: [PATCH 43/50] feat(connector_api): list disabled channels too Fixes https://emqx.atlassian.net/browse/EMQX-11583 --- .../emqx_connector/src/emqx_connector_api.erl | 16 +++++++-- .../test/emqx_connector_api_SUITE.erl | 33 +++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index af5721585..c55b13266 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -611,10 +611,11 @@ format_resource( type := Type, name := ConnectorName, raw_config := RawConf0, - resource_data := ResourceData + resource_data := ResourceData0 }, Node ) -> + ResourceData = lookup_channels(Type, ConnectorName, ResourceData0), RawConf = fill_defaults(Type, RawConf0), redact( maps:merge( @@ -627,14 +628,23 @@ format_resource( ) ). +lookup_channels(Type, Name, ResourceData0) -> + ConnectorResId = emqx_connector_resource:resource_id(Type, Name), + case emqx_resource:get_channels(ConnectorResId) of + {ok, Channels} -> + ResourceData0#{channels => maps:from_list(Channels)}; + {error, not_found} -> + ResourceData0#{channels => #{}} + end. + format_resource_data(ResData) -> - maps:fold(fun format_resource_data/3, #{}, maps:with([status, error, added_channels], ResData)). + maps:fold(fun format_resource_data/3, #{}, maps:with([status, error, channels], ResData)). format_resource_data(error, undefined, Result) -> Result; format_resource_data(error, Error, Result) -> Result#{status_reason => emqx_utils:readable_error_msg(Error)}; -format_resource_data(added_channels, Channels, Result) -> +format_resource_data(channels, Channels, Result) -> Result#{actions => lists:map(fun format_action/1, maps:keys(Channels))}; format_resource_data(K, V, Result) -> Result#{K => V}. diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index 92fd1de6d..dd6a0dec3 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -845,6 +845,39 @@ t_fail_delete_with_action(Config) -> ), ok. +t_list_disabled_channels(Config) -> + ConnectorParams = ?KAFKA_CONNECTOR(?CONNECTOR_NAME), + ?assertMatch( + {ok, 201, _}, + request_json( + post, + uri(["connectors"]), + ConnectorParams, + Config + ) + ), + ActionName = ?BRIDGE_NAME, + ActionParams = (?KAFKA_BRIDGE(ActionName))#{<<"enable">> := true}, + ?assertMatch( + {ok, 201, #{<<"enable">> := true}}, + request_json( + post, + uri(["actions"]), + ActionParams, + Config + ) + ), + ConnectorID = emqx_connector_resource:connector_id(?CONNECTOR_TYPE, ?CONNECTOR_NAME), + ?assertMatch( + {ok, 200, #{<<"actions">> := [ActionName]}}, + request_json( + get, + uri(["connectors", ConnectorID]), + Config + ) + ), + ok. + t_raw_config_response_defaults(Config) -> Params = maps:without([<<"enable">>, <<"resource_opts">>], ?KAFKA_CONNECTOR(?CONNECTOR_NAME)), ?assertMatch( From 4a71aa58ceb68ca1e388acb957bcb7d092acbd16 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 12 Dec 2023 10:04:15 -0300 Subject: [PATCH 44/50] fix(connector_api): return status reason when status is inconsistent Fixes https://emqx.atlassian.net/browse/EMQX-11581 --- .../emqx_connector/src/emqx_connector_api.erl | 14 ++- .../test/emqx_connector_api_SUITE.erl | 108 ++++++++++-------- 2 files changed, 75 insertions(+), 47 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index af5721585..26387b844 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -589,14 +589,24 @@ pick_connectors_by_id(Type, Name, ConnectorsAllNodes) -> format_connector_info([FirstConnector | _] = Connectors) -> Res = maps:remove(node, FirstConnector), NodeStatus = node_status(Connectors), - redact(Res#{ + StatusReason = first_status_reason(Connectors), + Info0 = Res#{ status => aggregate_status(NodeStatus), node_status => NodeStatus - }). + }, + Info = emqx_utils_maps:put_if(Info0, status_reason, StatusReason, StatusReason =/= undefined), + redact(Info). node_status(Connectors) -> [maps:with([node, status, status_reason], B) || B <- Connectors]. +first_status_reason(Connectors) -> + StatusReasons = [Reason || #{status_reason := Reason} <- Connectors, Reason =/= undefined], + case StatusReasons of + [Reason | _] -> Reason; + _ -> undefined + end. + aggregate_status(AllStatus) -> Head = fun([A | _]) -> A end, HeadVal = maps:get(status, Head(AllStatus), connecting), diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index 92fd1de6d..01f340662 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -23,6 +23,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/test_macros.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). -define(CONNECTOR_NAME, (atom_to_binary(?FUNCTION_NAME))). -define(RESOURCE(NAME, TYPE), #{ @@ -103,48 +104,6 @@ }). -define(KAFKA_BRIDGE(Name), ?KAFKA_BRIDGE(Name, ?CONNECTOR_NAME)). -%% -define(CONNECTOR_TYPE_MQTT, <<"mqtt">>). -%% -define(MQTT_CONNECTOR(SERVER, NAME), ?CONNECTOR(NAME, ?CONNECTOR_TYPE_MQTT)#{ -%% <<"server">> => SERVER, -%% <<"username">> => <<"user1">>, -%% <<"password">> => <<"">>, -%% <<"proto_ver">> => <<"v5">>, -%% <<"egress">> => #{ -%% <<"remote">> => #{ -%% <<"topic">> => <<"emqx/${topic}">>, -%% <<"qos">> => <<"${qos}">>, -%% <<"retain">> => false -%% } -%% } -%% }). -%% -define(MQTT_CONNECTOR(SERVER), ?MQTT_CONNECTOR(SERVER, <<"mqtt_egress_test_connector">>)). - -%% -define(CONNECTOR_TYPE_HTTP, <<"kafka_producer">>). -%% -define(HTTP_CONNECTOR(URL, NAME), ?CONNECTOR(NAME, ?CONNECTOR_TYPE_HTTP)#{ -%% <<"url">> => URL, -%% <<"local_topic">> => <<"emqx_webhook/#">>, -%% <<"method">> => <<"post">>, -%% <<"body">> => <<"${payload}">>, -%% <<"headers">> => #{ -%% % NOTE -%% % The Pascal-Case is important here. -%% % The reason is kinda ridiculous: `emqx_connector_resource:create_dry_run/2` converts -%% % connector config keys into atoms, and the atom 'Content-Type' exists in the ERTS -%% % when this happens (while the 'content-type' does not). -%% <<"Content-Type">> => <<"application/json">> -%% } -%% }). -%% -define(HTTP_CONNECTOR(URL), ?HTTP_CONNECTOR(URL, ?CONNECTOR_NAME)). - -%% -define(URL(PORT, PATH), -%% list_to_binary( -%% io_lib:format( -%% "http://localhost:~s/~s", -%% [integer_to_list(PORT), PATH] -%% ) -%% ) -%% ). - -define(APPSPECS, [ emqx_conf, emqx, @@ -178,11 +137,14 @@ groups() -> t_fail_delete_with_action, t_actions_field ], + ClusterOnlyTests = [ + t_inconsistent_state + ], ClusterLaterJoinOnlyTCs = [ % t_cluster_later_join_metrics ], [ - {single, [], AllTCs -- ClusterLaterJoinOnlyTCs}, + {single, [], (AllTCs -- ClusterLaterJoinOnlyTCs) -- ClusterOnlyTests}, {cluster_later_join, [], ClusterLaterJoinOnlyTCs}, {cluster, [], (AllTCs -- SingleOnlyTests) -- ClusterLaterJoinOnlyTCs} ]. @@ -268,6 +230,8 @@ init_mocks(_TestCase) -> fun (<<"connector:", ?CONNECTOR_TYPE_STR, ":bad_", _/binary>>, _C) -> {ok, bad_connector_state}; + (_I, #{bootstrap_hosts := <<"nope:9092">>}) -> + {ok, worst_connector_state}; (_I, _C) -> {ok, connector_state} end @@ -277,8 +241,17 @@ init_mocks(_TestCase) -> ?CONNECTOR_IMPL, on_get_status, fun - (_, bad_connector_state) -> connecting; - (_, _) -> connected + (_, bad_connector_state) -> + connecting; + (_, worst_connector_state) -> + {?status_disconnected, worst_connector_state, [ + #{ + host => <<"nope:9092">>, + reason => unresolvable_hostname + } + ]}; + (_, _) -> + connected end ), meck:expect(?CONNECTOR_IMPL, on_add_channel, 4, {ok, connector_state}), @@ -858,6 +831,51 @@ t_raw_config_response_defaults(Config) -> ), ok. +t_inconsistent_state(Config) -> + [_, Node2] = ?config(cluster_nodes, Config), + Params = ?KAFKA_CONNECTOR(?CONNECTOR_NAME), + ?assertMatch( + {ok, 201, #{<<"enable">> := true, <<"resource_opts">> := #{}}}, + request_json( + post, + uri(["connectors"]), + Params, + Config + ) + ), + BadParams = maps:without( + [<<"name">>, <<"type">>], + Params#{<<"bootstrap_hosts">> := <<"nope:9092">>} + ), + {ok, _} = erpc:call( + Node2, + emqx, + update_config, + [[connectors, ?CONNECTOR_TYPE, ?CONNECTOR_NAME], BadParams, #{}] + ), + + ConnectorID = emqx_connector_resource:connector_id(?CONNECTOR_TYPE, ?CONNECTOR_NAME), + ?assertMatch( + {ok, 200, #{ + <<"status">> := <<"inconsistent">>, + <<"node_status">> := [ + #{<<"status">> := <<"connected">>}, + #{ + <<"status">> := <<"disconnected">>, + <<"status_reason">> := _ + } + ], + <<"status_reason">> := _ + }}, + request_json( + get, + uri(["connectors", ConnectorID]), + Config + ) + ), + + ok. + %%% helpers listen_on_random_port() -> SockOpts = [binary, {active, false}, {packet, raw}, {reuseaddr, true}, {backlog, 1000}], From c73b371a7a7e18fee3d4349befda5bb87ca6068e Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 12 Dec 2023 17:37:19 +0800 Subject: [PATCH 45/50] feat: don't merge default headers if user already setting one --- apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl | 4 ++-- apps/emqx_conf/test/emqx_conf_schema_tests.erl | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl index f782e0e6c..ddcddabc6 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl @@ -281,12 +281,12 @@ parse_url(Url) -> end. convert_headers(Headers) -> - maps:merge(default_headers(), transform_header_name(Headers)). + transform_header_name(Headers). convert_headers_no_content_type(Headers) -> maps:without( [<<"content-type">>], - maps:merge(default_headers_no_content_type(), transform_header_name(Headers)) + transform_header_name(Headers) ). default_headers() -> diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 22f8c5575..6aafd99fc 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -358,7 +358,7 @@ authn_validations_test() -> Headers0 = authentication_headers(Res0), ?assertEqual(<<"application/json">>, maps:get(<<"content-type">>, Headers0)), %% accept from converter - ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers0)), + ?assertNot(maps:is_key(<<"accept">>, Headers0)), OKHttp = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"http://127.0.0.1:8080">>]), Conf1 = <>, @@ -366,7 +366,7 @@ authn_validations_test() -> {_, Res1} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap1, #{format => richmap}), Headers1 = authentication_headers(Res1), ?assertEqual(<<"application/json">>, maps:get(<<"content-type">>, Headers1), Headers1), - ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers1), Headers1), + ?assertNot(maps:is_key(<<"accept">>, Headers1)), DisableSSLWithHttps = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"https://127.0.0.1:8080">>]), Conf2 = <>, @@ -382,16 +382,16 @@ authn_validations_test() -> {_, Res3} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap3, #{format => richmap}), Headers3 = authentication_headers(Res3), %% remove the content-type header when get method - ?assertEqual(false, maps:is_key(<<"content-type">>, Headers3), Headers3), - ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers3), Headers3), + ?assertNot(maps:is_key(<<"content-type">>, Headers3), Headers3), + ?assertNot(maps:is_key(<<"accept">>, Headers3), Headers3), BadHeaderWithTuple = binary:replace(BadHeader, [<<"[">>, <<"]">>], <<"">>, [global]), Conf4 = <>, {ok, ConfMap4} = hocon:binary(Conf4, #{format => richmap}), {_, Res4} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap4, #{}), Headers4 = authentication_headers(Res4), - ?assertEqual(false, maps:is_key(<<"content-type">>, Headers4), Headers4), - ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers4), Headers4), + ?assertNot(maps:is_key(<<"content-type">>, Headers4), Headers4), + ?assertNot(maps:is_key(<<"accept">>, Headers4), Headers4), ok. %% erlfmt-ignore From 60d9c1113d3ab4fd94dc3c0fe2e8d69b8aacdd7d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 12 Dec 2023 16:01:16 +0800 Subject: [PATCH 46/50] fix: audit create_at/duration_ms filter not working --- apps/emqx_audit/src/emqx_audit.erl | 2 +- apps/emqx_audit/src/emqx_audit_api.erl | 10 +-- apps/emqx_audit/test/emqx_audit_api_SUITE.erl | 64 +++++++++++++++++-- 3 files changed, 65 insertions(+), 11 deletions(-) diff --git a/apps/emqx_audit/src/emqx_audit.erl b/apps/emqx_audit/src/emqx_audit.erl index ffbfaebb3..ad6afa9c4 100644 --- a/apps/emqx_audit/src/emqx_audit.erl +++ b/apps/emqx_audit/src/emqx_audit.erl @@ -119,7 +119,7 @@ log_to_db(Log) -> Audit0 = to_audit(Log), Audit = Audit0#?AUDIT{ node = node(), - created_at = erlang:system_time(microsecond) + created_at = erlang:system_time(millisecond) }, mria:dirty_write(?AUDIT, Audit). diff --git a/apps/emqx_audit/src/emqx_audit_api.erl b/apps/emqx_audit/src/emqx_audit_api.erl index a7fd8f4ad..303ba044b 100644 --- a/apps/emqx_audit/src/emqx_audit_api.erl +++ b/apps/emqx_audit/src/emqx_audit_api.erl @@ -33,7 +33,7 @@ {<<"gte_created_at">>, timestamp}, {<<"lte_created_at">>, timestamp}, {<<"gte_duration_ms">>, timestamp}, - {<<"lte_duration_ms">>, timestamp} + {<<"lte_duration_ms">>, integer} ]). -define(DISABLE_MSG, <<"Audit is disabled">>). @@ -290,16 +290,16 @@ gen_match_spec([{http_status_code, '=:=', T} | Qs], Audit, Conn) -> gen_match_spec([{http_method, '=:=', T} | Qs], Audit, Conn) -> gen_match_spec(Qs, Audit#?AUDIT{http_method = T}, Conn); gen_match_spec([{created_at, Hold, T} | Qs], Audit, Conn) -> - gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [{'$1', Hold, T} | Conn]); + gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [{Hold, '$1', T} | Conn]); gen_match_spec([{created_at, Hold1, T1, Hold2, T2} | Qs], Audit, Conn) -> gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [ - {'$1', Hold1, T1}, {'$1', Hold2, T2} | Conn + {Hold1, '$1', T1}, {Hold2, '$1', T2} | Conn ]); gen_match_spec([{duration_ms, Hold, T} | Qs], Audit, Conn) -> - gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [{'$2', Hold, T} | Conn]); + gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [{Hold, '$2', T} | Conn]); gen_match_spec([{duration_ms, Hold1, T1, Hold2, T2} | Qs], Audit, Conn) -> gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [ - {'$2', Hold1, T1}, {'$2', Hold2, T2} | Conn + {Hold1, '$2', T1}, {Hold2, '$2', T2} | Conn ]). format(Audit) -> diff --git a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl index 50b39d240..59dab550e 100644 --- a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl +++ b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl @@ -109,7 +109,7 @@ t_disabled(_) -> Size1 = mnesia:table_info(emqx_audit, size), {ok, Logs} = emqx_mgmt_api_configs_SUITE:get_config("log"), - Logs1 = emqx_utils_maps:deep_put([<<"audit">>, <<"max_filter_size">>], Logs, 100), + Logs1 = emqx_utils_maps:deep_put([<<"audit">>, <<"max_filter_size">>], Logs, 199), NewLogs = emqx_utils_maps:deep_put([<<"audit">>, <<"enable">>], Logs1, false), {ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", NewLogs), {ok, GetLog1} = emqx_mgmt_api_configs_SUITE:get_config("log"), @@ -139,6 +139,11 @@ t_disabled(_) -> ok. t_cli(_Config) -> + Size = mnesia:table_info(emqx_audit, size), + TimeInt = erlang:system_time(millisecond) - 10, + Time = integer_to_list(TimeInt), + DateStr = calendar:system_time_to_rfc3339(TimeInt, [{unit, millisecond}]), + Date = emqx_http_lib:uri_encode(DateStr), ok = emqx_ctl:run_command(["conf", "show", "log"]), AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), @@ -160,7 +165,7 @@ t_cli(_Config) -> Data ), - %% check filter + %% check cli filter {ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "from=cli", AuthHeader), #{<<"data">> := Data1} = emqx_utils_json:decode(Res1, [return_maps]), ?assertEqual(Data, Data1), @@ -168,10 +173,41 @@ t_cli(_Config) -> get, AuditPath, "from=erlang_console", AuthHeader ), ?assertMatch(#{<<"data">> := []}, emqx_utils_json:decode(Res2, [return_maps])), + + %% check created_at filter + {ok, Res3} = emqx_mgmt_api_test_util:request_api( + get, AuditPath, "gte_created_at=" ++ Time, AuthHeader + ), + #{<<"data">> := Data3} = emqx_utils_json:decode(Res3, [return_maps]), + ?assertEqual(1, erlang:length(Data3)), + {ok, Res31} = emqx_mgmt_api_test_util:request_api( + get, AuditPath, "gte_created_at=" ++ Date, AuthHeader + ), + ?assertEqual(Res3, Res31), + {ok, Res4} = emqx_mgmt_api_test_util:request_api( + get, AuditPath, "lte_created_at=" ++ Time, AuthHeader + ), + #{<<"data">> := Data4} = emqx_utils_json:decode(Res4, [return_maps]), + ?assertEqual(Size, erlang:length(Data4)), + {ok, Res41} = emqx_mgmt_api_test_util:request_api( + get, AuditPath, "lte_created_at=" ++ Date, AuthHeader + ), + ?assertEqual(Res4, Res41), + + %% check duration_ms filter + {ok, Res5} = emqx_mgmt_api_test_util:request_api( + get, AuditPath, "gte_duration_ms=0", AuthHeader + ), + #{<<"data">> := Data5} = emqx_utils_json:decode(Res5, [return_maps]), + ?assertEqual(Size + 1, erlang:length(Data5)), + {ok, Res6} = emqx_mgmt_api_test_util:request_api( + get, AuditPath, "lte_duration_ms=-1", AuthHeader + ), + ?assertMatch(#{<<"data">> := []}, emqx_utils_json:decode(Res6, [return_maps])), ok. t_max_size(_Config) -> - {ok, _} = emqx:update_config([log, audit, max_filter_size], 1000), + {ok, _} = emqx:update_config([log, audit, max_filter_size], 999), SizeFun = fun() -> AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), @@ -188,9 +224,14 @@ t_max_size(_Config) -> end, lists:duplicate(100, 1) ), - timer:sleep(110), + LogCount = wait_for_dirty_write_log_done(1500), Size1 = SizeFun(), - ?assert(Size1 - InitSize >= 100, {Size1, InitSize}), + ?assert(Size1 - InitSize >= 100, #{ + api => Size1, + init => InitSize, + log_size => LogCount, + config => emqx:get_config([log, audit, max_filter_size]) + }), {ok, _} = emqx:update_config([log, audit, max_filter_size], 10), %% wait for clean_expired timer:sleep(250), @@ -246,3 +287,16 @@ kickout_clients() -> {ok, Clients2} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), ClientsResponse2 = emqx_utils_json:decode(Clients2, [return_maps]), ?assertMatch(#{<<"data">> := []}, ClientsResponse2). + +wait_for_dirty_write_log_done(MaxMs) -> + Size = mnesia:table_info(emqx_audit, size), + wait_for_dirty_write_log_done(Size, MaxMs). + +wait_for_dirty_write_log_done(Size, RemainMs) when RemainMs =< 0 -> Size; +wait_for_dirty_write_log_done(Prev, RemainMs) -> + SleepMs = 100, + ct:sleep(SleepMs), + case mnesia:table_info(emqx_audit, size) of + Prev -> Prev; + New -> wait_for_dirty_write_log_done(New, RemainMs - SleepMs) + end. From 96d21d4dbe6094910af42a6f02c3b7bc6b40a429 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 13 Dec 2023 17:07:28 +0800 Subject: [PATCH 47/50] fix: ensure config update error text is readable --- .../src/emqx_mgmt_api_configs.erl | 8 ++++-- .../test/emqx_mgmt_api_configs_SUITE.erl | 27 +++++++++++++++---- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index d08bb9882..e07895ef0 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -354,8 +354,12 @@ configs(get, #{query_string := QueryStr, headers := Headers}, _Req) -> end; configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) -> case emqx_conf_cli:load_config(Conf, #{mode => Mode, log => none}) of - ok -> {200}; - {error, Msg} -> {400, #{<<"content-type">> => <<"text/plain">>}, Msg} + ok -> + {200}; + {error, MsgList} -> + JsonFun = fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end, + JsonMap = emqx_utils_maps:jsonable_map(maps:from_list(MsgList), JsonFun), + {400, #{<<"content-type">> => <<"text/plain">>}, JsonMap} end. find_suitable_accept(Headers, Preferences) when is_list(Preferences), length(Preferences) > 0 -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 8ff3c1794..2c1225c77 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -70,7 +70,7 @@ t_update(_Config) -> %% update failed ErrorSysMon = emqx_utils_maps:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"), ?assertMatch( - {error, {"HTTP/1.1", 400, _}}, + {error, {"HTTP/1.1", 400, "Bad Request"}}, update_config(<<"sysmon">>, ErrorSysMon) ), {ok, SysMon2} = get_config(<<"sysmon">>), @@ -328,6 +328,18 @@ t_configs_key(_Config) -> Log1 = emqx_utils_maps:deep_put([<<"log">>, <<"console">>, <<"level">>], Log, <<"error">>), ?assertEqual([], update_configs_with_binary(iolist_to_binary(hocon_pp:do(Log1, #{})))), ?assertEqual(<<"error">>, read_conf([<<"log">>, <<"console">>, <<"level">>])), + BadLog = emqx_utils_maps:deep_put([<<"log">>, <<"console">>, <<"level">>], Log, <<"erro1r">>), + {error, Error} = update_configs_with_binary(iolist_to_binary(hocon_pp:do(BadLog, #{}))), + ExpectError = #{ + <<"log">> => + #{ + <<"kind">> => <<"validation_error">>, + <<"path">> => <<"log.console.level">>, + <<"reason">> => <<"unable_to_convert_to_enum_symbol">>, + <<"value">> => <<"erro1r">> + } + }, + ?assertEqual(ExpectError, emqx_utils_json:decode(Error, [return_maps])), ok. t_get_configs_in_different_accept(_Config) -> @@ -348,7 +360,7 @@ t_get_configs_in_different_accept(_Config) -> end end, - %% returns text/palin if text/plain is acceptable + %% returns text/plain if text/plain is acceptable ?assertMatch({200, "text/plain", _}, Request(<<"text/plain">>)), ?assertMatch({200, "text/plain", _}, Request(<<"*/*">>)), ?assertMatch( @@ -416,9 +428,14 @@ update_configs_with_binary(Bin) -> Auth = emqx_mgmt_api_test_util:auth_header_(), Headers = [{"accept", "text/plain"}, Auth], case httpc:request(put, {Path, Headers, "text/plain", Bin}, [], []) of - {ok, {_, _, Res}} -> Res; - {ok, Res} -> Res; - Error -> Error + {ok, {{"HTTP/1.1", Code, _}, _Headers, Body}} when + Code >= 200 andalso Code =< 299 + -> + Body; + {ok, {{"HTTP/1.1", _Code, _}, _Headers, Body}} -> + {error, Body}; + Error -> + Error end. update_config(Name, Change) -> From d8796ea9859ffed52ed6f328a616d9f01c4060b8 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 13 Dec 2023 17:31:08 +0800 Subject: [PATCH 48/50] test: flaky audit log test --- apps/emqx_audit/test/emqx_audit_api_SUITE.erl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl index 59dab550e..e7e6c1268 100644 --- a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl +++ b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl @@ -208,6 +208,8 @@ t_cli(_Config) -> t_max_size(_Config) -> {ok, _} = emqx:update_config([log, audit, max_filter_size], 999), + %% Make sure this process is using latest max_filter_size. + ?assertEqual(ignore, gen_server:call(emqx_audit, whatever)), SizeFun = fun() -> AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), @@ -297,6 +299,9 @@ wait_for_dirty_write_log_done(Prev, RemainMs) -> SleepMs = 100, ct:sleep(SleepMs), case mnesia:table_info(emqx_audit, size) of - Prev -> Prev; - New -> wait_for_dirty_write_log_done(New, RemainMs - SleepMs) + Prev -> + ct:sleep(SleepMs * 2), + Prev; + New -> + wait_for_dirty_write_log_done(New, RemainMs - SleepMs) end. From b35765708d231efb037a05f1b1dd8d517c3a9c6a Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 14 Dec 2023 09:48:17 +0800 Subject: [PATCH 49/50] test: audit max_size flaky test --- apps/emqx_audit/test/emqx_audit_api_SUITE.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl index e7e6c1268..954292f9b 100644 --- a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl +++ b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl @@ -224,8 +224,9 @@ t_max_size(_Config) -> fun(_) -> ok = emqx_ctl:run_command(["conf", "show", "log"]) end, - lists:duplicate(100, 1) + lists:duplicate(110, 1) ), + _ = mnesia:dump_log(), LogCount = wait_for_dirty_write_log_done(1500), Size1 = SizeFun(), ?assert(Size1 - InitSize >= 100, #{ From f1bde41bf66e825e587dfbd50c75a19aaed33def Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 13 Dec 2023 16:37:34 +0800 Subject: [PATCH 50/50] fix(sso): Correctly handle wrapped passwords when updating --- apps/emqx/rebar.config | 2 +- apps/emqx/src/emqx_config.erl | 9 +++++---- apps/emqx/src/emqx_schema_secret.erl | 2 +- .../src/emqx_dashboard_sso_manager.erl | 8 +++++++- mix.exs | 2 +- rebar.config | 2 +- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index b5cf22798..763e3ace9 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -30,7 +30,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.9"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.17.0"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.0"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.2"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.3"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 2f2c711ef..617cb9b30 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -135,7 +135,8 @@ %% save the updated config to the emqx_override.conf file %% defaults to `true` persistent => boolean(), - override_to => local | cluster + override_to => local | cluster, + lazy_evaluator => fun((function()) -> term()) }. -type update_args() :: {update_cmd(), Opts :: update_opts()}. -type update_stage() :: pre_config_update | post_config_update. @@ -616,14 +617,14 @@ save_to_override_conf(true, RawConf, Opts) -> undefined -> ok; FileName -> - backup_and_write(FileName, hocon_pp:do(RawConf, #{})) + backup_and_write(FileName, hocon_pp:do(RawConf, Opts)) end; -save_to_override_conf(false, RawConf, _Opts) -> +save_to_override_conf(false, RawConf, Opts) -> case cluster_hocon_file() of undefined -> ok; FileName -> - backup_and_write(FileName, hocon_pp:do(RawConf, #{})) + backup_and_write(FileName, hocon_pp:do(RawConf, Opts)) end. %% @private This is the same human-readable timestamp format as diff --git a/apps/emqx/src/emqx_schema_secret.erl b/apps/emqx/src/emqx_schema_secret.erl index 635285ce7..bcce02797 100644 --- a/apps/emqx/src/emqx_schema_secret.erl +++ b/apps/emqx/src/emqx_schema_secret.erl @@ -23,7 +23,7 @@ -export([mk/1]). %% HOCON Schema API --export([convert_secret/2]). +-export([convert_secret/2, source/1]). %% @doc Secret value. -type t() :: binary(). diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl index ee22a48e8..378d6398c 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl @@ -208,7 +208,13 @@ start_backend_services() -> update_config(Backend, UpdateReq) -> %% we always make sure the valid configuration will update successfully, %% ignore the runtime error during its update - case emqx_conf:update(?MOD_KEY_PATH(Backend), UpdateReq, #{override_to => cluster}) of + case + emqx_conf:update( + ?MOD_KEY_PATH(Backend), + UpdateReq, + #{override_to => cluster, lazy_evaluator => fun emqx_schema_secret:source/1} + ) + of {ok, _UpdateResult} -> case lookup(Backend) of undefined -> diff --git a/mix.exs b/mix.exs index 26f49b318..338bf7812 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.40.2", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.40.3", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.3", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index eaf82f47b..661049f11 100644 --- a/rebar.config +++ b/rebar.config @@ -84,7 +84,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.2"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.3"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}