From ce2b159022c164673a700ab8fec4db1334613845 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 1 Aug 2023 16:48:45 +0300 Subject: [PATCH 001/119] chore(auth): make pre_hook_authenticate be part of emqx_access_control domain --- apps/emqx/include/emqx_access_control.hrl | 3 ++ apps/emqx/include/emqx_authentication.hrl | 2 +- apps/emqx/src/emqx_access_control.erl | 39 ++++++++++++++++++++++- apps/emqx/src/emqx_authentication.erl | 17 ---------- 4 files changed, 42 insertions(+), 19 deletions(-) diff --git a/apps/emqx/include/emqx_access_control.hrl b/apps/emqx/include/emqx_access_control.hrl index e840d2b4a..08146339b 100644 --- a/apps/emqx/include/emqx_access_control.hrl +++ b/apps/emqx/include/emqx_access_control.hrl @@ -32,3 +32,6 @@ -define(authz_action(PUBSUB, QOS), #{action_type := PUBSUB, qos := QOS}). -define(authz_action(PUBSUB), ?authz_action(PUBSUB, _)). -define(authz_action, ?authz_action(_)). + +-define(AUTHN_TRACE_TAG, "AUTHN"). + diff --git a/apps/emqx/include/emqx_authentication.hrl b/apps/emqx/include/emqx_authentication.hrl index 70b35a474..d668e9a54 100644 --- a/apps/emqx/include/emqx_authentication.hrl +++ b/apps/emqx/include/emqx_authentication.hrl @@ -18,8 +18,8 @@ -define(EMQX_AUTHENTICATION_HRL, true). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). --define(AUTHN_TRACE_TAG, "AUTHN"). -define(GLOBAL, 'mqtt:global'). -define(TRACE_AUTHN_PROVIDER(Msg), ?TRACE_AUTHN_PROVIDER(Msg, #{})). diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index 43669bf6c..a8ffadb44 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -17,6 +17,7 @@ -module(emqx_access_control). -include("emqx.hrl"). +-include("emqx_access_control.hrl"). -include("logger.hrl"). -export([ @@ -29,6 +30,14 @@ -compile(nowarn_export_all). -endif. +-define(TRACE_RESULT(Label, Tag, Result, Reason), begin + ?TRACE(Label, Tag, #{ + result => (Result), + reason => (Reason) + }), + Result +end). + %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -44,7 +53,7 @@ authenticate(Credential) -> %% if auth backend returning nothing but just 'ok' %% it means it's not a superuser, or there is no way to tell. NotSuperUser = #{is_superuser => false}, - case emqx_authentication:pre_hook_authenticate(Credential) of + case pre_hook_authenticate(Credential) of ok -> inc_authn_metrics(anonymous), {ok, NotSuperUser}; @@ -99,6 +108,34 @@ authorize(ClientInfo, Action, Topic) -> inc_authz_metrics(Result), Result. +%%-------------------------------------------------------------------- +%% Internal Functions +%%-------------------------------------------------------------------- + +-spec pre_hook_authenticate(emqx_types:clientinfo()) -> + ok | continue | {error, not_authorized}. +pre_hook_authenticate(#{enable_authn := false}) -> + ?TRACE_RESULT("pre_hook_authenticate", ?AUTHN_TRACE_TAG, ok, enable_authn_false); +pre_hook_authenticate(#{enable_authn := quick_deny_anonymous} = Credential) -> + case is_username_defined(Credential) of + true -> + continue; + false -> + ?TRACE_RESULT( + "pre_hook_authenticate", + ?AUTHN_TRACE_TAG, + {error, not_authorized}, + enable_authn_false + ) + end; +pre_hook_authenticate(_) -> + continue. + +is_username_defined(#{username := undefined}) -> false; +is_username_defined(#{username := <<>>}) -> false; +is_username_defined(#{username := _Username}) -> true; +is_username_defined(_) -> false. + check_authorization_cache(ClientInfo, Action, Topic) -> case emqx_authz_cache:get_authz_cache(Action, Topic) of not_found -> diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index cce789f24..92182f93a 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -29,11 +29,9 @@ -include_lib("stdlib/include/ms_transform.hrl"). -define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). --define(IS_UNDEFINED(X), (X =:= undefined orelse X =:= <<>>)). %% The authentication entrypoint. -export([ - pre_hook_authenticate/1, authenticate/2 ]). @@ -220,21 +218,6 @@ when %%------------------------------------------------------------------------------ %% Authenticate %%------------------------------------------------------------------------------ --spec pre_hook_authenticate(emqx_types:clientinfo()) -> - ok | continue | {error, not_authorized}. -pre_hook_authenticate(#{enable_authn := false}) -> - ?TRACE_RESULT("authentication_result", ok, enable_authn_false); -pre_hook_authenticate(#{enable_authn := quick_deny_anonymous} = Credential) -> - case maps:get(username, Credential, undefined) of - U when ?IS_UNDEFINED(U) -> - ?TRACE_RESULT( - "authentication_result", {error, not_authorized}, enable_authn_false - ); - _ -> - continue - end; -pre_hook_authenticate(_) -> - continue. authenticate(#{listener := Listener, protocol := Protocol} = Credential, AuthResult) -> case get_authenticators(Listener, global_chain(Protocol)) of From ca8c1e3ef87c5feff932e0a28dd456422bee9a12 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 1 Aug 2023 19:38:58 +0300 Subject: [PATCH 002/119] chore(auth): removed direct usages of authn from emqx core app --- apps/emqx/include/emqx_access_control.hrl | 4 + apps/emqx/include/emqx_hooks.hrl | 1 + .../emqx_authentication_listener_hooks.erl | 86 +++++++++++++++++++ apps/emqx/src/emqx_listeners.erl | 35 ++------ apps/emqx/test/emqx_broker_SUITE.erl | 31 +++---- 5 files changed, 109 insertions(+), 48 deletions(-) create mode 100644 apps/emqx/src/emqx_authentication_listener_hooks.erl diff --git a/apps/emqx/include/emqx_access_control.hrl b/apps/emqx/include/emqx_access_control.hrl index 08146339b..f7348899b 100644 --- a/apps/emqx/include/emqx_access_control.hrl +++ b/apps/emqx/include/emqx_access_control.hrl @@ -14,6 +14,9 @@ %% limitations under the License. %%-------------------------------------------------------------------- +-ifndef(EMQX_ACCESS_CONTROL_HRL). +-define(EMQX_ACCESS_CONTROL_HRL, true). + %% config root name all auth providers have to agree on. -define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME, "authorization"). -define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_ATOM, authorization). @@ -35,3 +38,4 @@ -define(AUTHN_TRACE_TAG, "AUTHN"). +-endif. diff --git a/apps/emqx/include/emqx_hooks.hrl b/apps/emqx/include/emqx_hooks.hrl index 2373b5928..dffc09fcd 100644 --- a/apps/emqx/include/emqx_hooks.hrl +++ b/apps/emqx/include/emqx_hooks.hrl @@ -29,6 +29,7 @@ -define(HP_RETAINER, 930). -define(HP_AUTO_SUB, 920). -define(HP_RULE_ENGINE, 900). + %% apps that can work with the republish action -define(HP_SLOW_SUB, 880). -define(HP_BRIDGE, 870). diff --git a/apps/emqx/src/emqx_authentication_listener_hooks.erl b/apps/emqx/src/emqx_authentication_listener_hooks.erl new file mode 100644 index 000000000..1109ca96b --- /dev/null +++ b/apps/emqx/src/emqx_authentication_listener_hooks.erl @@ -0,0 +1,86 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authentication_listener_hooks). + +-include_lib("emqx/include/emqx_hooks.hrl"). + +-export([ + on_listener_started/4, + on_listener_stopped/4, + on_listener_updated/4 +]). + +-export([ + load/0, + unload/0 +]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +load() -> + ok = emqx_hook:put('listener.started', {?MODULE, on_listener_started, []}, ?HP_AUTHN), + ok = emqx_hook:put('listener.stopped', {?MODULE, on_listener_stopped, []}, ?HP_AUTHN), + ok = emqx_hook:put('listener.updated', {?MODULE, on_listener_updated, []}, ?HP_AUTHN), + ok. + +unload() -> + ok = emqx_hooks:del('listener.started', {?MODULE, authenticate, []}), + ok = emqx_hooks:del('listener.stopped', {?MODULE, authenticate, []}), + ok = emqx_hooks:del('listener.updated', {?MODULE, authenticate, []}), + ok. + +%%-------------------------------------------------------------------- +%% Hooks +%%-------------------------------------------------------------------- + +on_listener_started(Type, Name, Conf, ok) -> + recreate_authenticators(Type, Name, Conf); +on_listener_started(_Type, _Name, _Conf, _Error) -> + ok. + +on_listener_updated(Type, Name, {_OldConf, NewConf}, ok) -> + recreate_authenticators(Type, Name, NewConf); +on_listener_updated(_Type, _Name, _Conf, _Error) -> + ok. + +on_listener_stopped(Type, Name, _OldConf, ok) -> + _ = emqx_authentication:delete_chain(emqx_listeners:listener_id(Type, Name)), + ok; +on_listener_stopped(_Type, _Name, _Conf, _Error) -> + ok. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +recreate_authenticators(Type, Name, Conf) -> + Chain = emqx_listeners:listener_id(Type, Name), + _ = emqx_authentication:delete_chain(Chain), + do_create_authneticators(Chain, maps:get(authentication, Conf, [])). + +do_create_authneticators(Chain, [AuthN | T]) -> + case emqx_authentication:create_authenticator(Chain, AuthN) of + {ok, _} -> + do_create_authneticators(Chain, T); + Error -> + _ = emqx_authentication:delete_chain(Chain), + {ok, Error} + end; +do_create_authneticators(_Chain, []) -> + ok. diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 964873e53..5d641de39 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -531,41 +531,18 @@ post_config_update(_Path, _Request, _NewConf, _OldConf, _AppEnvs) -> ok. create_listener(Type, Name, NewConf) -> - Res = start_listener(Type, Name, NewConf), - recreate_authenticators(Res, Type, Name, NewConf). - -recreate_authenticators(ok, Type, Name, Conf) -> - Chain = listener_id(Type, Name), - _ = emqx_authentication:delete_chain(Chain), - do_create_authneticators(Chain, maps:get(authentication, Conf, [])); -recreate_authenticators(Error, _Type, _Name, _NewConf) -> - Error. - -do_create_authneticators(Chain, [AuthN | T]) -> - case emqx_authentication:create_authenticator(Chain, AuthN) of - {ok, _} -> - do_create_authneticators(Chain, T); - Error -> - _ = emqx_authentication:delete_chain(Chain), - Error - end; -do_create_authneticators(_Chain, []) -> - ok. + StartRes = start_listener(Type, Name, NewConf), + emqx_hooks:run_fold('listener.started', [Type, Name, NewConf], StartRes). remove_listener(Type, Name, OldConf) -> ok = unregister_ocsp_stapling_refresh(Type, Name), - case stop_listener(Type, Name, OldConf) of - ok -> - _ = emqx_authentication:delete_chain(listener_id(Type, Name)), - ok; - Err -> - Err - end. + StopRes = stop_listener(Type, Name, OldConf), + emqx_hooks:run_fold('listener.stopped', [Type, Name, OldConf], StopRes). update_listener(Type, Name, {OldConf, NewConf}) -> ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf), - Res = restart_listener(Type, Name, {OldConf, NewConf}), - recreate_authenticators(Res, Type, Name, NewConf). + RestartRes = restart_listener(Type, Name, {OldConf, NewConf}), + emqx_hooks:run_fold('listener.restarted', [Type, Name, {OldConf, NewConf}], RestartRes). perform_listener_changes([]) -> ok; diff --git a/apps/emqx/test/emqx_broker_SUITE.erl b/apps/emqx/test/emqx_broker_SUITE.erl index 6e03971a5..ba5438641 100644 --- a/apps/emqx/test/emqx_broker_SUITE.erl +++ b/apps/emqx/test/emqx_broker_SUITE.erl @@ -26,6 +26,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_hooks.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). all() -> @@ -680,28 +681,17 @@ t_connect_client_never_negative({'end', _Config}) -> t_connack_auth_error({init, Config}) -> process_flag(trap_exit, true), - ChainName = 'mqtt:global', - AuthenticatorConfig = #{ - enable => true, - mechanism => password_based, - backend => built_in_database, - user_id_type => username, - password_hash_algorithm => #{ - name => plain, - salt_position => disable - }, - user_group => <<"global:mqtt">> - }, - ok = emqx_authentication:register_providers( - [{{password_based, built_in_database}, emqx_authentication_SUITE}] + emqx_hooks:put( + 'client.authenticate', + {?MODULE, authenticate_deny, []}, + ?HP_AUTHN ), - emqx_authentication:initialize_authentication(ChainName, AuthenticatorConfig), Config; t_connack_auth_error({'end', _Config}) -> - ChainName = 'mqtt:global', - AuthenticatorID = <<"password_based:built_in_database">>, - ok = emqx_authentication:deregister_provider({password_based, built_in_database}), - ok = emqx_authentication:delete_authenticator(ChainName, AuthenticatorID), + emqx_hooks:del( + 'client.authenticate', + {?MODULE, authenticate_deny, []} + ), ok; t_connack_auth_error(Config) when is_list(Config) -> %% MQTT 3.1 @@ -733,6 +723,9 @@ t_handle_in_empty_client_subscribe_hook(Config) when is_list(Config) -> emqtt:disconnect(C) end. +authenticate_deny(_Credentials, _Default) -> + {stop, {error, bad_username_or_password}}. + wait_for_events(Action, Kinds) -> wait_for_events(Action, Kinds, 500). From 8213aa42c9a5290a373b1b896117e4a28191f41f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 1 Aug 2023 21:15:04 +0300 Subject: [PATCH 003/119] chore(auth): move authn leftovers to the authn app --- apps/emqx/include/emqx.hrl | 16 ---- apps/emqx/include/emqx_access_control.hrl | 1 - apps/emqx/src/emqx_broker_sup.erl | 12 +-- apps/emqx/src/emqx_schema.erl | 46 +---------- apps/emqx/src/emqx_schema_hooks.erl | 78 +++++++++++++++++++ apps/emqx/test/emqx_common_test_helpers.erl | 2 +- apps/emqx/test/emqx_cth_suite.erl | 2 +- .../include/emqx_authentication.hrl | 16 ++-- apps/emqx_authn/include/emqx_authn.hrl | 2 +- .../src/emqx_authentication.erl | 15 +++- .../src/emqx_authentication_config.erl | 2 +- .../emqx_authentication_listener_hooks.erl | 0 .../src/emqx_authentication_sup.erl | 0 apps/emqx_authn/src/emqx_authn_api.erl | 1 - apps/emqx_authn/src/emqx_authn_app.erl | 2 +- apps/emqx_authn/src/emqx_authn_schema.erl | 43 +++++++++- apps/emqx_authn/src/emqx_authn_sup.erl | 12 ++- .../src/emqx_authn_user_import_api.erl | 1 - .../test/emqx_authentication_SUITE.erl | 0 apps/emqx_conf/src/emqx_conf_cli.erl | 4 +- apps/emqx_conf/src/emqx_conf_schema.erl | 18 +++-- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_gateway/src/emqx_gateway_conf.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_http.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_schema.erl | 2 +- rel/i18n/emqx_authn_schema.hocon | 27 +++++++ rel/i18n/emqx_schema.hocon | 24 ------ 27 files changed, 200 insertions(+), 132 deletions(-) create mode 100644 apps/emqx/src/emqx_schema_hooks.erl rename apps/{emqx => emqx_authn}/include/emqx_authentication.hrl (85%) rename apps/{emqx => emqx_authn}/src/emqx_authentication.erl (99%) rename apps/{emqx => emqx_authn}/src/emqx_authentication_config.erl (99%) rename apps/{emqx => emqx_authn}/src/emqx_authentication_listener_hooks.erl (100%) rename apps/{emqx => emqx_authn}/src/emqx_authentication_sup.erl (100%) rename apps/{emqx => emqx_authn}/test/emqx_authentication_SUITE.erl (100%) diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index ac9d297de..2cba12365 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -123,20 +123,4 @@ until :: integer() }). -%%-------------------------------------------------------------------- -%% Authentication -%%-------------------------------------------------------------------- - --record(authenticator, { - id :: binary(), - provider :: module(), - enable :: boolean(), - state :: map() -}). - --record(chain, { - name :: atom(), - authenticators :: [#authenticator{}] -}). - -endif. diff --git a/apps/emqx/include/emqx_access_control.hrl b/apps/emqx/include/emqx_access_control.hrl index f7348899b..65a159dd6 100644 --- a/apps/emqx/include/emqx_access_control.hrl +++ b/apps/emqx/include/emqx_access_control.hrl @@ -17,7 +17,6 @@ -ifndef(EMQX_ACCESS_CONTROL_HRL). -define(EMQX_ACCESS_CONTROL_HRL, true). -%% config root name all auth providers have to agree on. -define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME, "authorization"). -define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_ATOM, authorization). -define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY, <<"authorization">>). diff --git a/apps/emqx/src/emqx_broker_sup.erl b/apps/emqx/src/emqx_broker_sup.erl index ac2fe587c..74baf5674 100644 --- a/apps/emqx/src/emqx_broker_sup.erl +++ b/apps/emqx/src/emqx_broker_sup.erl @@ -49,16 +49,6 @@ init([]) -> modules => [emqx_shared_sub] }, - %% Authentication - AuthNSup = #{ - id => emqx_authentication_sup, - start => {emqx_authentication_sup, start_link, []}, - restart => permanent, - shutdown => infinity, - type => supervisor, - modules => [emqx_authentication_sup] - }, - %% Broker helper Helper = #{ id => helper, @@ -69,4 +59,4 @@ init([]) -> modules => [emqx_broker_helper] }, - {ok, {{one_for_all, 0, 1}, [BrokerPool, SharedSub, AuthNSup, Helper]}}. + {ok, {{one_for_all, 0, 1}, [BrokerPool, SharedSub, Helper]}}. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index e6bff790e..3f4507a3c 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -24,7 +24,6 @@ -elvis([{elvis_style, invalid_dynamic_call, disable}]). -include("emqx_schema.hrl"). --include("emqx_authentication.hrl"). -include("emqx_access_control.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -216,7 +215,6 @@ roots(high) -> importance => ?IMPORTANCE_HIDDEN } )}, - {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication(global)}, %% NOTE: authorization schema here is only to keep emqx app pure %% the full schema for EMQX node is injected in emqx_conf_schema. {?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME, @@ -224,7 +222,7 @@ roots(high) -> ref(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME), #{importance => ?IMPORTANCE_HIDDEN} )} - ]; + ] ++ emqx_schema_hooks:injection_point('roots.high'); roots(medium) -> [ {"broker", @@ -1750,11 +1748,8 @@ mqtt_listener(Bind) -> desc => ?DESC(mqtt_listener_proxy_protocol_timeout), default => <<"3s">> } - )}, - {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, (authentication(listener))#{ - importance => ?IMPORTANCE_HIDDEN - }} - ]. + )} + ] ++ emqx_schema_hooks:injection_point('mqtt.listener'). base_listener(Bind) -> [ @@ -2762,41 +2757,6 @@ str(B) when is_binary(B) -> str(S) when is_list(S) -> S. -authentication(Which) -> - {Importance, Desc} = - case Which of - global -> - %% For root level authentication, it is recommended to configure - %% from the dashboard or API. - %% Hence it's considered a low-importance when it comes to - %% configuration importance. - {?IMPORTANCE_LOW, ?DESC(global_authentication)}; - listener -> - {?IMPORTANCE_HIDDEN, ?DESC(listener_authentication)} - end, - %% poor man's dependency injection - %% this is due to the fact that authn is implemented outside of 'emqx' app. - %% so it can not be a part of emqx_schema since 'emqx' app is supposed to - %% work standalone. - Type = - case persistent_term:get(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, undefined) of - undefined -> - hoconsc:array(typerefl:map()); - Module -> - Module:root_type() - end, - hoconsc:mk(Type, #{ - desc => Desc, - converter => fun ensure_array/2, - default => [], - importance => Importance - }). - -%% the older version schema allows individual element (instead of a chain) in config -ensure_array(undefined, _) -> undefined; -ensure_array(L, _) when is_list(L) -> L; -ensure_array(M, _) -> [M]. - -spec qos() -> typerefl:type(). qos() -> typerefl:alias("qos", typerefl:union([0, 1, 2])). diff --git a/apps/emqx/src/emqx_schema_hooks.erl b/apps/emqx/src/emqx_schema_hooks.erl new file mode 100644 index 000000000..5a0d9a813 --- /dev/null +++ b/apps/emqx/src/emqx_schema_hooks.erl @@ -0,0 +1,78 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_schema_hooks). + +-type hookpoint() :: atom(). + +-callback injected_fields() -> + #{ + hookpoint() => [hocon_schema:field()] + }. +-optional_callbacks([injected_fields/0]). + +-define(HOOKPOINT_PT_KEY(POINT_NAME), {?MODULE, fields, POINT_NAME}). +-define(MODULE_PT_KEY(MOD_NAME), {?MODULE, mod, MOD_NAME}). + +-export([ + inject_fields/3, + injection_point/1, + + inject_fields_from_mod/1 +]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +injection_point(PointName) -> + InjectedFields = persistent_term:get(?HOOKPOINT_PT_KEY(PointName), #{}), + lists:concat(maps:values(InjectedFields)). + +inject_fields(PointName, Name, Fields) -> + Key = ?HOOKPOINT_PT_KEY(PointName), + InjectedFields = persistent_term:get(Key, #{}), + persistent_term:put(Key, InjectedFields#{Name => Fields}). + +%%-------------------------------------------------------------------- +%% Internal API +%%-------------------------------------------------------------------- + +inject_fields_from_mod(Module) -> + case persistent_term:get(?MODULE_PT_KEY(Module), false) of + false -> + persistent_term:put(?MODULE_PT_KEY(Module), true), + do_inject_fields_from_mod(Module); + true -> + ok + end. + +do_inject_fields_from_mod(Module) -> + _ = Module:module_info(), + case erlang:function_exported(Module, injected_fields, 0) of + true -> + do_inject_fields_from_mod(Module, Module:injected_fields()); + false -> + ok + end. + +do_inject_fields_from_mod(Module, HookFields) -> + maps:foreach( + fun(PointName, Fields) -> + inject_fields(PointName, Module, Fields) + end, + HookFields + ). diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 7f1fe4628..edf21c4f6 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -16,7 +16,7 @@ -module(emqx_common_test_helpers). --include_lib("emqx/include/emqx_authentication.hrl"). +-include_lib("emqx_authn/include/emqx_authentication.hrl"). -type special_config_handler() :: fun(). diff --git a/apps/emqx/test/emqx_cth_suite.erl b/apps/emqx/test/emqx_cth_suite.erl index 1ae6ceded..853281648 100644 --- a/apps/emqx/test/emqx_cth_suite.erl +++ b/apps/emqx/test/emqx_cth_suite.erl @@ -17,7 +17,7 @@ -module(emqx_cth_suite). -include_lib("common_test/include/ct.hrl"). --include_lib("emqx/include/emqx_authentication.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -export([start/2]). -export([stop/1]). diff --git a/apps/emqx/include/emqx_authentication.hrl b/apps/emqx_authn/include/emqx_authentication.hrl similarity index 85% rename from apps/emqx/include/emqx_authentication.hrl rename to apps/emqx_authn/include/emqx_authentication.hrl index d668e9a54..a367af291 100644 --- a/apps/emqx/include/emqx_authentication.hrl +++ b/apps/emqx_authn/include/emqx_authentication.hrl @@ -20,6 +20,11 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_access_control.hrl"). +%% config root name all auth providers have to agree on. +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, "authentication"). +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, authentication). +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, <<"authentication">>). + -define(GLOBAL, 'mqtt:global'). -define(TRACE_AUTHN_PROVIDER(Msg), ?TRACE_AUTHN_PROVIDER(Msg, #{})). @@ -31,17 +36,6 @@ -define(TRACE_AUTHN(Msg, Meta), ?TRACE_AUTHN(debug, Msg, Meta)). -define(TRACE_AUTHN(Level, Msg, Meta), ?TRACE(Level, ?AUTHN_TRACE_TAG, Msg, Meta)). -%% config root name all auth providers have to agree on. --define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, "authentication"). --define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, authentication). --define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, <<"authentication">>). - -%% key to a persistent term which stores a module name in order to inject -%% schema module at run-time to keep emqx app's compile time purity. -%% see emqx_schema.erl for more details -%% and emqx_conf_schema for an examples --define(EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, emqx_authentication_schema_module). - %% authentication move cmd -define(CMD_MOVE_FRONT, front). -define(CMD_MOVE_REAR, rear). diff --git a/apps/emqx_authn/include/emqx_authn.hrl b/apps/emqx_authn/include/emqx_authn.hrl index 601b161d5..9574d092f 100644 --- a/apps/emqx_authn/include/emqx_authn.hrl +++ b/apps/emqx_authn/include/emqx_authn.hrl @@ -17,7 +17,7 @@ -ifndef(EMQX_AUTHN_HRL). -define(EMQX_AUTHN_HRL, true). --include_lib("emqx/include/emqx_authentication.hrl"). +-include_lib("emqx_authentication.hrl"). -define(APP, emqx_authn). diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx_authn/src/emqx_authentication.erl similarity index 99% rename from apps/emqx/src/emqx_authentication.erl rename to apps/emqx_authn/src/emqx_authentication.erl index 92182f93a..ed02cc67a 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx_authn/src/emqx_authentication.erl @@ -22,14 +22,25 @@ -behaviour(gen_server). --include("emqx.hrl"). --include("logger.hrl"). -include("emqx_authentication.hrl"). +-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). -define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). +-record(authenticator, { + id :: binary(), + provider :: module(), + enable :: boolean(), + state :: map() +}). + +-record(chain, { + name :: atom(), + authenticators :: [#authenticator{}] +}). + %% The authentication entrypoint. -export([ authenticate/2 diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx_authn/src/emqx_authentication_config.erl similarity index 99% rename from apps/emqx/src/emqx_authentication_config.erl rename to apps/emqx_authn/src/emqx_authentication_config.erl index 96718d611..0137e7b68 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx_authn/src/emqx_authentication_config.erl @@ -37,8 +37,8 @@ -export_type([config/0]). --include("logger.hrl"). -include("emqx_authentication.hrl"). +-include_lib("emqx/include/logger.hrl"). -type parsed_config() :: #{ mechanism := atom(), diff --git a/apps/emqx/src/emqx_authentication_listener_hooks.erl b/apps/emqx_authn/src/emqx_authentication_listener_hooks.erl similarity index 100% rename from apps/emqx/src/emqx_authentication_listener_hooks.erl rename to apps/emqx_authn/src/emqx_authentication_listener_hooks.erl diff --git a/apps/emqx/src/emqx_authentication_sup.erl b/apps/emqx_authn/src/emqx_authentication_sup.erl similarity index 100% rename from apps/emqx/src/emqx_authentication_sup.erl rename to apps/emqx_authn/src/emqx_authentication_sup.erl diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index fa9f6c820..ce4647110 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -21,7 +21,6 @@ -include("emqx_authn.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). --include_lib("emqx/include/emqx_authentication.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -import(hoconsc, [mk/2, ref/1, ref/2]). diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index 5d4be5f41..9ba6f5fbc 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -26,7 +26,7 @@ stop/1 ]). --include_lib("emqx/include/emqx_authentication.hrl"). +-include_lib("emqx_authentication.hrl"). -dialyzer({nowarn_function, [start/2]}). diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index a7cdaac5f..2a12899aa 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -19,6 +19,12 @@ -elvis([{elvis_style, invalid_dynamic_call, disable}]). -include_lib("hocon/include/hoconsc.hrl"). -include("emqx_authn.hrl"). +-include("emqx_authentication.hrl"). + +-behaviour(emqx_schema_hooks). +-export([ + injected_fields/0 +]). -export([ common_fields/0, @@ -28,13 +34,18 @@ fields/1, authenticator_type/0, authenticator_type_without_scram/0, - root_type/0, mechanism/1, backend/1 ]). roots() -> []. +injected_fields() -> + #{ + 'roots.high' => global_auth_fields(), + 'mqtt.listener' => mqtt_listener_auth_fields() + }. + tags() -> [<<"Authentication">>]. @@ -121,12 +132,36 @@ try_select_union_member(Module, Value) -> Module:refs() end. -%% authn is a core functionality however implemented outside of emqx app -%% in emqx_schema, 'authentication' is a map() type which is to allow -%% EMQX more pluggable. root_type() -> hoconsc:array(authenticator_type()). +global_auth_fields() -> + [ + {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, + hoconsc:mk(root_type(), #{ + desc => ?DESC(global_authentication), + converter => fun ensure_array/2, + default => [], + importance => ?IMPORTANCE_LOW + })} + ]. + +mqtt_listener_auth_fields() -> + [ + {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, + hoconsc:mk(root_type(), #{ + desc => ?DESC(listener_authentication), + converter => fun ensure_array/2, + default => [], + importance => ?IMPORTANCE_HIDDEN + })} + ]. + +%% the older version schema allows individual element (instead of a chain) in config +ensure_array(undefined, _) -> undefined; +ensure_array(L, _) when is_list(L) -> L; +ensure_array(M, _) -> [M]. + mechanism(Name) -> ?HOCON( Name, diff --git a/apps/emqx_authn/src/emqx_authn_sup.erl b/apps/emqx_authn/src/emqx_authn_sup.erl index 635bd7323..211ebd518 100644 --- a/apps/emqx_authn/src/emqx_authn_sup.erl +++ b/apps/emqx_authn/src/emqx_authn_sup.erl @@ -27,5 +27,15 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - ChildSpecs = [], + AuthNSup = #{ + id => emqx_authentication_sup, + start => {emqx_authentication_sup, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [emqx_authentication_sup] + }, + + ChildSpecs = [AuthNSup], + {ok, {{one_for_one, 10, 10}, ChildSpecs}}. diff --git a/apps/emqx_authn/src/emqx_authn_user_import_api.erl b/apps/emqx_authn/src/emqx_authn_user_import_api.erl index 30836d3ba..f9d4208e6 100644 --- a/apps/emqx_authn/src/emqx_authn_user_import_api.erl +++ b/apps/emqx_authn/src/emqx_authn_user_import_api.erl @@ -20,7 +20,6 @@ -include("emqx_authn.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_authentication.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -import(emqx_dashboard_swagger, [error_codes/2]). diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx_authn/test/emqx_authentication_SUITE.erl similarity index 100% rename from apps/emqx/test/emqx_authentication_SUITE.erl rename to apps/emqx_authn/test/emqx_authentication_SUITE.erl diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index fde3059d3..8a4bb131f 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -16,8 +16,8 @@ -module(emqx_conf_cli). -include("emqx_conf.hrl"). --include_lib("emqx/include/emqx_access_control.hrl"). --include_lib("emqx/include/emqx_authentication.hrl"). +-include_lib("emqx_authn/include/emqx_authentication.hrl"). +-include_lib("emqx/include/logger.hrl"). -export([ load/0, diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 246f36f41..14cd4f17c 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -22,9 +22,9 @@ -dialyzer(no_unused). -dialyzer(no_fail_call). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). --include_lib("emqx/include/emqx_authentication.hrl"). -type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all. -type file() :: string(). @@ -66,6 +66,10 @@ emqx_otel_schema, emqx_mgmt_api_key_schema ]). +-define(INJECTING_CONFIGS, [ + emqx_authn_schema +]). + %% 1 million default ports counter -define(DEFAULT_MAX_PORTS, 1024 * 1024). @@ -76,11 +80,7 @@ tags() -> [<<"EMQX">>]. roots() -> - PtKey = ?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, - case persistent_term:get(PtKey, undefined) of - undefined -> persistent_term:put(PtKey, emqx_authn_schema); - _ -> ok - end, + ok = ensure_fields_injected(), emqx_schema_high_prio_roots() ++ [ {"node", @@ -1434,3 +1434,9 @@ ensure_unicode_path(Path, _) when is_list(Path) -> Path; ensure_unicode_path(Path, _) -> throw({"not_string", Path}). + +ensure_fields_injected() -> + lists:foreach( + fun(Module) -> emqx_schema_hooks:inject_fields_from_mod(Module) end, + ?INJECTING_CONFIGS + ). diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index b5fe5e100..582269ce6 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.22"}, + {vsn, "0.1.23"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, emqx, emqx_authn, emqx_ctl]}, diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index 2a64a6914..480633652 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -71,7 +71,7 @@ ]). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_authentication.hrl"). +-include_lib("emqx_authn/include/emqx_authentication.hrl"). -define(AUTHN_BIN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY). -type atom_or_bin() :: atom() | binary(). diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index 2186ac3d7..997539e7d 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -19,7 +19,7 @@ -include("include/emqx_gateway.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_authentication.hrl"). +-include_lib("emqx_authn/include/emqx_authentication.hrl"). -define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index b43f4ba98..8d9cc5a11 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -24,9 +24,9 @@ -dialyzer(no_unused). -dialyzer(no_fail_call). --include_lib("emqx/include/emqx_authentication.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). +-include_lib("emqx_authn/include/emqx_authentication.hrl"). -type ip_port() :: tuple() | integer(). -type duration() :: non_neg_integer(). diff --git a/rel/i18n/emqx_authn_schema.hocon b/rel/i18n/emqx_authn_schema.hocon index 98263ca49..a1910f95b 100644 --- a/rel/i18n/emqx_authn_schema.hocon +++ b/rel/i18n/emqx_authn_schema.hocon @@ -1,5 +1,32 @@ emqx_authn_schema { +global_authentication.desc: +"""Default authentication configs for all MQTT listeners. + +For per-listener overrides see authentication in listener configs + +This option can be configured with: +
    +
  • []: The default value, it allows *ALL* logins
    • +
  • one: For example {enable:true,backend:"built_in_database",mechanism="password_based"}
    • +
  • chain: An array of structs.
    • +
+ +When a chain is configured, the login credentials are checked against the backends per the configured order, until an 'allow' or 'deny' decision can be made. + +If there is no decision after a full chain exhaustion, the login is rejected.""" + +global_authentication.label: +"""Global authentication""" + +listener_authentication.desc: +"""Per-listener authentication override. +Authentication can be one single authenticator instance or a chain of authenticators as an array. +When authenticating a login (username, client ID, etc.) the authenticators are checked in the configured order.""" + +listener_authentication.label: +"""Per-listener authentication override""" + backend.desc: """Backend type.""" diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 64de73b24..57b24f57d 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -532,22 +532,6 @@ mqtt_server_keepalive.desc: mqtt_server_keepalive.label: """Server Keep Alive""" -global_authentication.desc: -"""Default authentication configs for all MQTT listeners. - -For per-listener overrides see authentication in listener configs - -This option can be configured with: -
    -
  • []: The default value, it allows *ALL* logins
    • -
  • one: For example {enable:true,backend:"built_in_database",mechanism="password_based"}
    • -
  • chain: An array of structs.
    • -
- -When a chain is configured, the login credentials are checked against the backends per the configured order, until an 'allow' or 'deny' decision can be made. - -If there is no decision after a full chain exhaustion, the login is rejected.""" - fields_mqtt_quic_listener_load_balancing_mode.desc: """0: Disabled, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED. default: 0""" @@ -1103,14 +1087,6 @@ See: https://erlang.org/doc/man/inet.html#setopts-2""" fields_tcp_opts_active_n.label: """active_n""" -listener_authentication.desc: -"""Per-listener authentication override. -Authentication can be one single authenticator instance or a chain of authenticators as an array. -When authenticating a login (username, client ID, etc.) the authenticators are checked in the configured order.""" - -listener_authentication.label: -"""Per-listener authentication override""" - fields_trace_payload_encode.desc: """Determine the format of the payload format in the trace file.
`text`: Text-based protocol or plain text protocol. From 128aa19d474cc993a9c1c42a8d455a8fe1a09f65 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 3 Aug 2023 10:43:33 +0300 Subject: [PATCH 004/119] chore(auth): improve emqx_config_handler to replace adhoc config updates --- apps/emqx/src/emqx_config_handler.erl | 293 +++++++++++++++--- apps/emqx/src/emqx_listeners.erl | 29 +- apps/emqx/src/emqx_schema_hooks.erl | 36 +++ apps/emqx/test/emqx_common_test_helpers.erl | 4 +- apps/emqx/test/emqx_config_handler_SUITE.erl | 90 +++++- apps/emqx/test/emqx_cth_suite.erl | 4 +- apps/emqx_authn/rebar.config | 2 + apps/emqx_authn/src/emqx_authentication.erl | 7 + .../src/emqx_authentication_config.erl | 32 +- .../emqx_authentication_listener_hooks.erl | 86 ----- apps/emqx_authn/src/emqx_authn_app.erl | 3 +- .../test/emqx_authentication_SUITE.erl | 47 ++- apps/emqx_conf/rebar.config | 5 +- apps/emqx_gateway/rebar.config | 3 +- apps/emqx_machine/rebar.config | 1 + 15 files changed, 451 insertions(+), 191 deletions(-) delete mode 100644 apps/emqx_authn/src/emqx_authentication_listener_hooks.erl diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index 96690c26e..f68992e4e 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -53,11 +53,17 @@ -optional_callbacks([ pre_config_update/3, - post_config_update/5 + propagated_pre_config_update/3, + post_config_update/5, + propagated_post_config_update/5 ]). -callback pre_config_update([atom()], emqx_config:update_request(), emqx_config:raw_config()) -> - {ok, emqx_config:update_request()} | {error, term()}. + ok | {ok, emqx_config:update_request()} | {error, term()}. +-callback propagated_pre_config_update( + [atom()], emqx_config:update_request(), emqx_config:raw_config() +) -> + ok | {error, term()}. -callback post_config_update( [atom()], @@ -68,6 +74,15 @@ ) -> ok | {ok, Result :: any()} | {error, Reason :: term()}. +-callback propagated_post_config_update( + [atom()], + emqx_config:update_request(), + emqx_config:config(), + emqx_config:config(), + emqx_config:app_envs() +) -> + ok | {ok, Result :: any()} | {error, Reason :: term()}. + -type state() :: #{handlers := any()}. start_link() -> @@ -244,7 +259,13 @@ do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) -> do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq, []). do_update_config([], Handlers, OldRawConf, UpdateReq, ConfKeyPath) -> - call_pre_config_update(Handlers, OldRawConf, UpdateReq, ConfKeyPath); + call_pre_config_update(#{ + handlers => Handlers, + old_raw_conf => OldRawConf, + update_req => UpdateReq, + conf_key_path => ConfKeyPath, + callback => pre_config_update + }); do_update_config( [ConfKey | SubConfKeyPath], Handlers, @@ -331,15 +352,16 @@ do_post_config_update( Result, ConfKeyPath ) -> - call_post_config_update( - Handlers, - OldConf, - NewConf, - AppEnvs, - up_req(UpdateArgs), - Result, - ConfKeyPath - ); + call_post_config_update(#{ + handlers => Handlers, + old_conf => OldConf, + new_conf => NewConf, + app_envs => AppEnvs, + update_req => up_req(UpdateArgs), + result => Result, + conf_key_path => ConfKeyPath, + callback => post_config_update + }); do_post_config_update( [ConfKey | SubConfKeyPath], Handlers, @@ -365,7 +387,7 @@ do_post_config_update( ConfKeyPath ). -get_sub_handlers(ConfKey, Handlers) -> +get_sub_handlers(ConfKey, Handlers) when is_atom(ConfKey) -> case maps:find(ConfKey, Handlers) of error -> maps:get(?WKEY, Handlers, #{}); {ok, SubHandlers} -> SubHandlers @@ -377,57 +399,231 @@ get_sub_config(ConfKey, Conf) when is_map(Conf) -> get_sub_config(_, _Conf) -> undefined. -call_pre_config_update(#{?MOD := HandlerName}, OldRawConf, UpdateReq, ConfKeyPath) -> - case erlang:function_exported(HandlerName, pre_config_update, 3) of +call_pre_config_update(Ctx) -> + case call_proper_pre_config_update(Ctx) of + {ok, NewUpdateReq} -> + case + propagate_pre_config_updates_to_subconf(Ctx#{ + update_req => NewUpdateReq + }) + of + {ok, _} -> + {ok, NewUpdateReq}; + {error, _} = Error -> + Error + end; + {error, _} = Error -> + Error + end. + +call_proper_pre_config_update( + #{ + handlers := #{?MOD := Module}, + callback := Callback, + update_req := UpdateReq, + old_raw_conf := OldRawConf + } = Ctx +) -> + case erlang:function_exported(Module, Callback, 3) of true -> - case HandlerName:pre_config_update(ConfKeyPath, UpdateReq, OldRawConf) of - {ok, NewUpdateReq} -> {ok, NewUpdateReq}; - {error, Reason} -> {error, {pre_config_update, HandlerName, Reason}} + case apply_pre_config_update(Module, Ctx) of + {ok, NewUpdateReq} -> + {ok, NewUpdateReq}; + ok -> + {ok, UpdateReq}; + {error, Reason} -> + {error, {pre_config_update, Module, Reason}} end; false -> merge_to_old_config(UpdateReq, OldRawConf) end; -call_pre_config_update(_Handlers, OldRawConf, UpdateReq, _ConfKeyPath) -> - merge_to_old_config(UpdateReq, OldRawConf). - -call_post_config_update( - #{?MOD := HandlerName}, - OldConf, - NewConf, - AppEnvs, - UpdateReq, - Result, - ConfKeyPath +call_proper_pre_config_update( + #{update_req := UpdateReq} ) -> - case erlang:function_exported(HandlerName, post_config_update, 5) of - true -> + {ok, UpdateReq}. + +apply_pre_config_update(Module, #{ + conf_key_path := ConfKeyPath, + update_req := UpdateReq, + old_raw_conf := OldRawConf, + callback := Callback +}) -> + Module:Callback( + ConfKeyPath, UpdateReq, OldRawConf + ). + +propagate_pre_config_updates_to_subconf( + #{handlers := #{?WKEY := _}} = Ctx +) -> + propagate_pre_config_updates_to_subconf_wkey(Ctx); +propagate_pre_config_updates_to_subconf( + #{handlers := Handlers} = Ctx +) -> + Keys = maps:keys(maps:without([?MOD], Handlers)), + propagate_pre_config_updates_to_subconf_keys(Keys, Ctx). + +propagate_pre_config_updates_to_subconf_wkey( + #{ + update_req := UpdateReq, + old_raw_conf := OldRawConf + } = Ctx +) -> + Keys = propagate_keys(UpdateReq, OldRawConf), + propagate_pre_config_updates_to_subconf_keys(Keys, Ctx). + +propagate_pre_config_updates_to_subconf_keys([], #{update_req := UpdateReq}) -> + {ok, UpdateReq}; +propagate_pre_config_updates_to_subconf_keys([Key | Keys], Ctx) -> + case propagate_pre_config_updates_to_subconf_key(Key, Ctx) of + ok -> + propagate_pre_config_updates_to_subconf_keys(Keys, Ctx); + {error, _} = Error -> + Error + end. + +propagate_pre_config_updates_to_subconf_key( + Key, + #{ + handlers := Handlers, + old_raw_conf := OldRawConf, + update_req := UpdateReq, + conf_key_path := ConfKeyPath + } = Ctx +) -> + AtomKey = atom(Key), + SubHandlers = get_sub_handlers(AtomKey, Handlers), + BinKey = bin(Key), + SubUpdateReq = get_sub_config(BinKey, UpdateReq), + SubOldConf = get_sub_config(BinKey, OldRawConf), + SubConfKeyPath = ConfKeyPath ++ [AtomKey], + case {SubOldConf, SubUpdateReq} of + {undefined, undefined} -> + ok; + {_, _} -> case - HandlerName:post_config_update( - ConfKeyPath, - UpdateReq, - NewConf, - OldConf, - AppEnvs - ) + call_pre_config_update(Ctx#{ + handlers := SubHandlers, + old_raw_conf := SubOldConf, + update_req := SubUpdateReq, + conf_key_path := SubConfKeyPath, + callback := propagated_pre_config_update + }) of + {ok, _SubNewConf1} -> + ok; + {error, _} = Error -> + Error + end + end. + +call_post_config_update(#{handlers := Handlers} = Ctx) -> + case call_proper_post_config_update(Ctx) of + {ok, Result} -> + SubHandlers = maps:without([?MOD], Handlers), + propagate_post_config_updates_to_subconf(Ctx#{ + handlers := SubHandlers, + callback := propagated_post_config_update, + result := Result + }); + {error, _} = Error -> + Error + end. + +call_proper_post_config_update( + #{ + handlers := #{?MOD := Module}, + callback := Callback, + result := Result + } = Ctx +) -> + case erlang:function_exported(Module, Callback, 5) of + true -> + case apply_post_config_update(Module, Ctx) of ok -> {ok, Result}; - {ok, Result1} -> {ok, Result#{HandlerName => Result1}}; - {error, Reason} -> {error, {post_config_update, HandlerName, Reason}} + {ok, Result1} -> {ok, Result#{Module => Result1}}; + {error, Reason} -> {error, {post_config_update, Module, Reason}} end; false -> {ok, Result} end; -call_post_config_update( - _Handlers, - _OldConf, - _NewConf, - _AppEnvs, - _UpdateReq, - Result, - _ConfKeyPath +call_proper_post_config_update( + #{result := Result} = _Ctx ) -> {ok, Result}. +apply_post_config_update(Module, #{ + conf_key_path := ConfKeyPath, + update_req := UpdateReq, + new_conf := NewConf, + old_conf := OldConf, + app_envs := AppEnvs, + callback := Callback +}) -> + Module:Callback( + ConfKeyPath, + UpdateReq, + NewConf, + OldConf, + AppEnvs + ). + +propagate_post_config_updates_to_subconf( + #{handlers := #{?WKEY := _}} = Ctx +) -> + propagate_post_config_updates_to_subconf_wkey(Ctx); +propagate_post_config_updates_to_subconf( + #{handlers := Handlers} = Ctx +) -> + Keys = maps:keys(Handlers), + propagate_post_config_updates_to_subconf_keys(Keys, Ctx). + +propagate_post_config_updates_to_subconf_wkey( + #{ + old_conf := OldConf, + new_conf := NewConf + } = Ctx +) -> + Keys = propagate_keys(OldConf, NewConf), + propagate_post_config_updates_to_subconf_keys(Keys, Ctx). +propagate_post_config_updates_to_subconf_keys([], #{result := Result}) -> + {ok, Result}; +propagate_post_config_updates_to_subconf_keys([Key | Keys], Ctx) -> + case propagate_post_config_updates_to_subconf_key(Key, Ctx) of + {ok, Result1} -> + propagate_post_config_updates_to_subconf_keys(Keys, Ctx#{result := Result1}); + Error -> + Error + end. + +propagate_keys(OldConf, NewConf) -> + sets:to_list(sets:union(propagate_keys(OldConf), propagate_keys(NewConf))). + +propagate_keys(Conf) when is_map(Conf) -> sets:from_list(maps:keys(Conf), [{version, 2}]); +propagate_keys(_) -> sets:new([{version, 2}]). + +propagate_post_config_updates_to_subconf_key( + Key, + #{ + handlers := Handlers, + new_conf := NewConf, + old_conf := OldConf, + result := Result, + conf_key_path := ConfKeyPath + } = Ctx +) -> + SubHandlers = maps:get(Key, Handlers, maps:get(?WKEY, Handlers, undefined)), + SubNewConf = get_sub_config(Key, NewConf), + SubOldConf = get_sub_config(Key, OldConf), + SubConfKeyPath = ConfKeyPath ++ [Key], + call_post_config_update(Ctx#{ + handlers := SubHandlers, + new_conf := SubNewConf, + old_conf := SubOldConf, + result := Result, + conf_key_path := SubConfKeyPath, + callback := propagated_post_config_update + }). + %% The default callback of config handlers %% the behaviour is overwriting the old config if: %% 1. the old config is undefined @@ -517,6 +713,7 @@ remove_empty_leaf(KeyPath, Handlers) -> end. assert_callback_function(Mod) -> + _ = Mod:module_info(), case erlang:function_exported(Mod, pre_config_update, 3) orelse erlang:function_exported(Mod, post_config_update, 5) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 5d641de39..b95169d3c 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -531,18 +531,15 @@ post_config_update(_Path, _Request, _NewConf, _OldConf, _AppEnvs) -> ok. create_listener(Type, Name, NewConf) -> - StartRes = start_listener(Type, Name, NewConf), - emqx_hooks:run_fold('listener.started', [Type, Name, NewConf], StartRes). + start_listener(Type, Name, NewConf). remove_listener(Type, Name, OldConf) -> ok = unregister_ocsp_stapling_refresh(Type, Name), - StopRes = stop_listener(Type, Name, OldConf), - emqx_hooks:run_fold('listener.stopped', [Type, Name, OldConf], StopRes). + stop_listener(Type, Name, OldConf). update_listener(Type, Name, {OldConf, NewConf}) -> ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf), - RestartRes = restart_listener(Type, Name, {OldConf, NewConf}), - emqx_hooks:run_fold('listener.restarted', [Type, Name, {OldConf, NewConf}], RestartRes). + restart_listener(Type, Name, {OldConf, NewConf}). perform_listener_changes([]) -> ok; @@ -824,10 +821,9 @@ convert_certs(ListenerConf) -> fun(Type, Listeners0, Acc) -> Listeners1 = maps:fold( - fun(Name, Conf, Acc1) -> - Conf1 = convert_certs(Type, Name, Conf), - Conf2 = convert_authn_certs(Type, Name, Conf1), - Acc1#{Name => Conf2} + fun(Name, Conf0, Acc1) -> + Conf1 = convert_certs(Type, Name, Conf0), + Acc1#{Name => Conf1} end, #{}, Listeners0 @@ -850,19 +846,6 @@ convert_certs(Type, Name, Conf) -> throw({bad_ssl_config, Reason}) end. -convert_authn_certs(Type, Name, #{<<"authentication">> := AuthNList} = Conf) -> - ChainName = listener_id(Type, Name), - AuthNList1 = lists:map( - fun(AuthN) -> - CertsDir = emqx_authentication_config:certs_dir(ChainName, AuthN), - emqx_authentication_config:convert_certs(CertsDir, AuthN) - end, - AuthNList - ), - Conf#{<<"authentication">> => AuthNList1}; -convert_authn_certs(_Type, _Name, Conf) -> - Conf. - filter_stacktrace({Reason, _Stacktrace}) -> Reason; filter_stacktrace(Reason) -> Reason. diff --git a/apps/emqx/src/emqx_schema_hooks.erl b/apps/emqx/src/emqx_schema_hooks.erl index 5a0d9a813..59fb1588b 100644 --- a/apps/emqx/src/emqx_schema_hooks.erl +++ b/apps/emqx/src/emqx_schema_hooks.erl @@ -34,6 +34,12 @@ inject_fields_from_mod/1 ]). +%% for tests +-export([ + erase_injections/0, + any_injections/0 +]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -60,6 +66,36 @@ inject_fields_from_mod(Module) -> ok end. +erase_injections() -> + lists:foreach( + fun + ({?HOOKPOINT_PT_KEY(_) = Key, _}) -> + persistent_term:erase(Key); + ({?MODULE_PT_KEY(_) = Key, _}) -> + persistent_term:erase(Key); + (_) -> + ok + end, + persistent_term:get() + ). + +any_injections() -> + lists:any( + fun + ({?HOOKPOINT_PT_KEY(_), _}) -> + true; + ({?MODULE_PT_KEY(_), _}) -> + true; + (_) -> + false + end, + persistent_term:get() + ). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + do_inject_fields_from_mod(Module) -> _ = Module:module_info(), case erlang:function_exported(Module, injected_fields, 0) of diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index edf21c4f6..3645fa06b 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -16,8 +16,6 @@ -module(emqx_common_test_helpers). --include_lib("emqx_authn/include/emqx_authentication.hrl"). - -type special_config_handler() :: fun(). -type apps() :: list(atom()). @@ -351,7 +349,7 @@ stop_apps(Apps, Opts) -> %% to avoid inter-suite flakiness application:unset_env(emqx, config_loader), application:unset_env(emqx, boot_modules), - persistent_term:erase(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY), + emqx_schema_hooks:erase_injections(), case Opts of #{erase_all_configs := false} -> %% FIXME: this means inter-suite or inter-test dependencies diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index b13da79f6..af2b4996c 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -26,7 +26,8 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -all() -> emqx_common_test_helpers:all(?MODULE). +all() -> + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> emqx_common_test_helpers:boot_modules(all), @@ -239,6 +240,62 @@ t_post_update_error(_Config) -> ), ok. +t_post_update_propagate_error_wkey(_Config) -> + Conf0 = emqx_config:get_raw([sysmon]), + Conf1 = emqx_utils_maps:deep_put([<<"os">>, <<"sysmem_high_watermark">>], Conf0, <<"60%">>), + callback_error( + [ + [sysmon, '?', sysmem_high_watermark], + [sysmon] + ], + [sysmon], + Conf1, + {error, {post_config_update, ?MODULE, post_config_update_error}} + ), + ok. + +t_post_update_propagate_error_key(_Config) -> + Conf0 = emqx_config:get_raw([sysmon]), + Conf1 = emqx_utils_maps:deep_put([<<"os">>, <<"sysmem_high_watermark">>], Conf0, <<"60%">>), + callback_error( + [ + [sysmon, os, sysmem_high_watermark], + [sysmon] + ], + [sysmon], + Conf1, + {error, {post_config_update, ?MODULE, post_config_update_error}} + ), + ok. + +t_pre_update_propagate_error_wkey(_Config) -> + Conf0 = emqx_config:get_raw([sysmon]), + Conf1 = emqx_utils_maps:deep_put([<<"os">>, <<"mem_check_interval">>], Conf0, <<"70s">>), + callback_error( + [ + [sysmon, '?', mem_check_interval], + [sysmon] + ], + [sysmon], + Conf1, + {error, {pre_config_update, ?MODULE, pre_config_update_error}} + ), + ok. + +t_pre_update_propagate_error_key(_Config) -> + Conf0 = emqx_config:get_raw([sysmon]), + Conf1 = emqx_utils_maps:deep_put([<<"os">>, <<"mem_check_interval">>], Conf0, <<"70s">>), + callback_error( + [ + [sysmon, os, mem_check_interval], + [sysmon] + ], + [sysmon], + Conf1, + {error, {pre_config_update, ?MODULE, pre_config_update_error}} + ), + ok. + t_handler_root() -> %% Don't rely on default emqx_config_handler's merge behaviour. RootKey = [], @@ -295,6 +352,11 @@ pre_config_update([sysmon, os, sysmem_high_watermark], UpdateReq, _RawConf) -> pre_config_update([sysmon, os, mem_check_interval], _UpdateReq, _RawConf) -> {error, pre_config_update_error}. +propagated_pre_config_update([sysmon, os, mem_check_interval], _UpdateReq, _RawConf) -> + {error, pre_config_update_error}; +propagated_pre_config_update(_ConfKeyPath, _UpdateReq, _RawConf) -> + ok. + post_config_update([sysmon], _UpdateReq, _NewConf, _OldConf, _AppEnvs) -> {ok, ok}; post_config_update([sysmon, os], _UpdateReq, _NewConf, _OldConf, _AppEnvs) -> @@ -308,6 +370,13 @@ post_config_update([sysmon, os, cpu_high_watermark], _UpdateReq, _NewConf, _OldC post_config_update([sysmon, os, sysmem_high_watermark], _UpdateReq, _NewConf, _OldConf, _AppEnvs) -> {error, post_config_update_error}. +propagated_post_config_update( + [sysmon, os, sysmem_high_watermark], _UpdateReq, _NewConf, _OldConf, _AppEnvs +) -> + {error, post_config_update_error}; +propagated_post_config_update(_ConfKeyPath, _UpdateReq, _NewConf, _OldConf, _AppEnvs) -> + ok. + wait_for_new_pid() -> case erlang:whereis(emqx_config_handler) of undefined -> @@ -318,10 +387,16 @@ wait_for_new_pid() -> end. callback_error(FailedPath, Update, ExpectError) -> + callback_error([FailedPath], FailedPath, Update, ExpectError). + +callback_error(Paths, UpdatePath, Update, ExpectError) -> + ok = lists:foreach( + fun(Path) -> emqx_config_handler:add_handler(Path, ?MODULE) end, + Paths + ), Opts = #{rawconf_with_defaults => true}, - ok = emqx_config_handler:add_handler(FailedPath, ?MODULE), - Old = emqx:get_raw_config(FailedPath, undefined), - Error = emqx:update_config(FailedPath, Update, Opts), + Old = emqx:get_raw_config(UpdatePath, undefined), + Error = emqx:update_config(UpdatePath, Update, Opts), case ExpectError of {error, {post_config_update, ?MODULE, post_config_update_error}} -> ?assertMatch( @@ -330,7 +405,10 @@ callback_error(FailedPath, Update, ExpectError) -> _ -> ?assertEqual(ExpectError, Error) end, - New = emqx:get_raw_config(FailedPath, undefined), + New = emqx:get_raw_config(UpdatePath, undefined), ?assertEqual(Old, New), - ok = emqx_config_handler:remove_handler(FailedPath), + ok = lists:foreach( + fun(Path) -> emqx_config_handler:remove_handler(Path) end, + Paths + ), ok. diff --git a/apps/emqx/test/emqx_cth_suite.erl b/apps/emqx/test/emqx_cth_suite.erl index 853281648..61f87e0ea 100644 --- a/apps/emqx/test/emqx_cth_suite.erl +++ b/apps/emqx/test/emqx_cth_suite.erl @@ -347,12 +347,12 @@ stop_apps(Apps) -> verify_clean_suite_state(#{work_dir := WorkDir}) -> {ok, []} = file:list_dir(WorkDir), - none = persistent_term:get(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, none), + false = emqx_schema_hooks:any_injections(), [] = emqx_config:get_root_names(), ok. clean_suite_state() -> - _ = persistent_term:erase(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY), + _ = emqx_schema_hooks:erase_injections(), _ = emqx_config:erase_all(), ok. diff --git a/apps/emqx_authn/rebar.config b/apps/emqx_authn/rebar.config index 932a1ff77..5bc8d3e91 100644 --- a/apps/emqx_authn/rebar.config +++ b/apps/emqx_authn/rebar.config @@ -34,4 +34,6 @@ {cover_opts, [verbose]}. {cover_export_enabled, true}. +{erl_first_files, ["src/emqx_authentication.erl"]}. + {project_plugins, [erlfmt]}. diff --git a/apps/emqx_authn/src/emqx_authentication.erl b/apps/emqx_authn/src/emqx_authentication.erl index ed02cc67a..5b49f5d28 100644 --- a/apps/emqx_authn/src/emqx_authentication.erl +++ b/apps/emqx_authn/src/emqx_authentication.erl @@ -60,6 +60,7 @@ register_providers/1, deregister_provider/1, deregister_providers/1, + providers/0, delete_chain/1, lookup_chain/1, list_chains/0, @@ -331,6 +332,10 @@ deregister_providers(AuthNTypes) when is_list(AuthNTypes) -> deregister_provider(AuthNType) -> deregister_providers([AuthNType]). +-spec providers() -> [{authn_type(), module()}]. +providers() -> + call(providers). + -spec delete_chain(chain_name()) -> ok | {error, term()}. delete_chain(Name) -> call({delete_chain, Name}). @@ -463,6 +468,8 @@ handle_call( end; handle_call({deregister_providers, AuthNTypes}, _From, #{providers := Providers} = State) -> reply(ok, State#{providers := maps:without(AuthNTypes, Providers)}); +handle_call(providers, _From, #{providers := Providers} = State) -> + reply(maps:to_list(Providers), State); handle_call({delete_chain, ChainName}, _From, State) -> UpdateFun = fun(Chain) -> {_MatchedIDs, NewChain} = do_delete_authenticators(fun(_) -> true end, Chain), diff --git a/apps/emqx_authn/src/emqx_authentication_config.erl b/apps/emqx_authn/src/emqx_authentication_config.erl index 0137e7b68..13b9214de 100644 --- a/apps/emqx_authn/src/emqx_authentication_config.erl +++ b/apps/emqx_authn/src/emqx_authentication_config.erl @@ -21,7 +21,9 @@ -export([ pre_config_update/3, - post_config_update/5 + post_config_update/5, + propagated_pre_config_update/3, + propagated_post_config_update/5 ]). -export([ @@ -37,8 +39,8 @@ -export_type([config/0]). +-include("logger.hrl"). -include("emqx_authentication.hrl"). --include_lib("emqx/include/logger.hrl"). -type parsed_config() :: #{ mechanism := atom(), @@ -145,6 +147,12 @@ do_pre_config_update(Paths, NewConfig, _OldConfig) -> || New <- to_list(NewConfig) ]}. +-spec propagated_pre_config_update(list(atom()), update_request(), emqx_config:raw_config()) -> + ok | {error, term()}. +propagated_pre_config_update(Paths, NewConfig, OldConfig) -> + {ok, _} = do_pre_config_update(Paths, NewConfig, OldConfig), + ok. + -spec post_config_update( list(atom()), update_request(), @@ -203,6 +211,26 @@ do_post_config_update(Paths, _UpdateReq, NewConfig0, OldConfig0, _AppEnvs) -> ok = emqx_authentication:reorder_authenticator(ChainName, NewIds), ok. +-spec propagated_post_config_update( + list(atom()), + update_request(), + map() | list() | undefined, + emqx_config:raw_config(), + emqx_config:app_envs() +) -> + ok | {ok, map()} | {error, term()}. + +propagated_post_config_update(Paths, UpdateReq, NewConfig, OldConfig, AppEnvs) -> + ok = post_config_update(Paths, UpdateReq, NewConfig, OldConfig, AppEnvs), + ChainName = chain_name(Paths), + ok = maybe_delete_chain(ChainName, NewConfig), + ok. + +maybe_delete_chain(ChainName, undefined) -> + ok = emqx_authentication:delete_chain(ChainName); +maybe_delete_chain(_ChainName, _NewConfig) -> + ok. + %% create new authenticators and update existing ones create_or_update_authenticators(OldIds, ChainName, NewConfig) -> lists:foreach( diff --git a/apps/emqx_authn/src/emqx_authentication_listener_hooks.erl b/apps/emqx_authn/src/emqx_authentication_listener_hooks.erl deleted file mode 100644 index 1109ca96b..000000000 --- a/apps/emqx_authn/src/emqx_authentication_listener_hooks.erl +++ /dev/null @@ -1,86 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_authentication_listener_hooks). - --include_lib("emqx/include/emqx_hooks.hrl"). - --export([ - on_listener_started/4, - on_listener_stopped/4, - on_listener_updated/4 -]). - --export([ - load/0, - unload/0 -]). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -load() -> - ok = emqx_hook:put('listener.started', {?MODULE, on_listener_started, []}, ?HP_AUTHN), - ok = emqx_hook:put('listener.stopped', {?MODULE, on_listener_stopped, []}, ?HP_AUTHN), - ok = emqx_hook:put('listener.updated', {?MODULE, on_listener_updated, []}, ?HP_AUTHN), - ok. - -unload() -> - ok = emqx_hooks:del('listener.started', {?MODULE, authenticate, []}), - ok = emqx_hooks:del('listener.stopped', {?MODULE, authenticate, []}), - ok = emqx_hooks:del('listener.updated', {?MODULE, authenticate, []}), - ok. - -%%-------------------------------------------------------------------- -%% Hooks -%%-------------------------------------------------------------------- - -on_listener_started(Type, Name, Conf, ok) -> - recreate_authenticators(Type, Name, Conf); -on_listener_started(_Type, _Name, _Conf, _Error) -> - ok. - -on_listener_updated(Type, Name, {_OldConf, NewConf}, ok) -> - recreate_authenticators(Type, Name, NewConf); -on_listener_updated(_Type, _Name, _Conf, _Error) -> - ok. - -on_listener_stopped(Type, Name, _OldConf, ok) -> - _ = emqx_authentication:delete_chain(emqx_listeners:listener_id(Type, Name)), - ok; -on_listener_stopped(_Type, _Name, _Conf, _Error) -> - ok. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -recreate_authenticators(Type, Name, Conf) -> - Chain = emqx_listeners:listener_id(Type, Name), - _ = emqx_authentication:delete_chain(Chain), - do_create_authneticators(Chain, maps:get(authentication, Conf, [])). - -do_create_authneticators(Chain, [AuthN | T]) -> - case emqx_authentication:create_authenticator(Chain, AuthN) of - {ok, _} -> - do_create_authneticators(Chain, T); - Error -> - _ = emqx_authentication:delete_chain(Chain), - {ok, Error} - end; -do_create_authneticators(_Chain, []) -> - ok. diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index 9ba6f5fbc..689f6619a 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -35,8 +35,7 @@ %%------------------------------------------------------------------------------ start(_StartType, _StartArgs) -> - %% required by test cases, ensure the injection of - %% EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY + %% required by test cases, ensure the injection of schema _ = emqx_conf_schema:roots(), ok = mria_rlog:wait_for_shards([?AUTH_SHARD], infinity), {ok, Sup} = emqx_authn_sup:start_link(), diff --git a/apps/emqx_authn/test/emqx_authentication_SUITE.erl b/apps/emqx_authn/test/emqx_authentication_SUITE.erl index fb73a3fc1..dfeeb458e 100644 --- a/apps/emqx_authn/test/emqx_authentication_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authentication_SUITE.erl @@ -94,19 +94,19 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - LogLevel = emqx_logger:get_primary_log_level(), - ok = emqx_logger:set_log_level(debug), - application:set_env(ekka, strict_mode, true), - emqx_config:erase_all(), - emqx_common_test_helpers:stop_apps([]), - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - [{log_level, LogLevel} | Config]. + Apps = emqx_cth_suite:start( + [ + emqx, + emqx_conf, + emqx_authn + ], + #{work_dir => ?config(priv_dir)} + ), + ok = deregister_providers(), + [{apps, Apps} | Config]. end_per_suite(Config) -> - emqx_common_test_helpers:stop_apps([]), - LogLevel = ?config(log_level), - emqx_logger:set_log_level(LogLevel), + emqx_cth_suite:stop(?config(apps)), ok. init_per_testcase(Case, Config) -> @@ -302,15 +302,20 @@ t_update_config(Config) when is_list(Config) -> ok = register_provider(?config("auth1"), ?MODULE), ok = register_provider(?config("auth2"), ?MODULE), Global = ?config(global), + %% We mocked provider implementation, but did't mock the schema + %% so we should provide full config AuthenticatorConfig1 = #{ - mechanism => password_based, - backend => built_in_database, - enable => true + <<"mechanism">> => <<"password_based">>, + <<"backend">> => <<"built_in_database">>, + <<"enable">> => true }, AuthenticatorConfig2 = #{ - mechanism => password_based, - backend => mysql, - enable => true + <<"mechanism">> => <<"password_based">>, + <<"backend">> => <<"mysql">>, + <<"query">> => <<"SELECT password_hash, salt FROM users WHERE username = ?">>, + <<"server">> => <<"127.0.0.1:5432">>, + <<"database">> => <<"emqx">>, + <<"enable">> => true }, ID1 = <<"password_based:built_in_database">>, ID2 = <<"password_based:mysql">>, @@ -580,3 +585,11 @@ certs(Certs) -> register_provider(Type, Module) -> ok = ?AUTHN:register_providers([{Type, Module}]). + +deregister_providers() -> + lists:foreach( + fun({Type, _Module}) -> + ok = ?AUTHN:deregister_provider(Type) + end, + lists:flatten([?AUTHN:providers()]) + ). diff --git a/apps/emqx_conf/rebar.config b/apps/emqx_conf/rebar.config index c947932a0..1d2f23bd0 100644 --- a/apps/emqx_conf/rebar.config +++ b/apps/emqx_conf/rebar.config @@ -1,7 +1,10 @@ %% -*- mode: erlang -*- {erl_opts, [debug_info]}. -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_authn, {path, "../emqx_authn"}} +]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_gateway/rebar.config b/apps/emqx_gateway/rebar.config index 2340a2dd8..e78c8a44b 100644 --- a/apps/emqx_gateway/rebar.config +++ b/apps/emqx_gateway/rebar.config @@ -2,5 +2,6 @@ {erl_opts, [debug_info]}. {deps, [ {emqx, {path, "../emqx"}}, - {emqx_utils, {path, "../emqx_utils"}} + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_authn, {path, "../emqx_authn"}} ]}. diff --git a/apps/emqx_machine/rebar.config b/apps/emqx_machine/rebar.config index 53b7bec13..8953b54a7 100644 --- a/apps/emqx_machine/rebar.config +++ b/apps/emqx_machine/rebar.config @@ -3,6 +3,7 @@ {deps, [ {emqx, {path, "../emqx"}}, {emqx_dashboard, {path, "../emqx_dashboard"}}, + {emqx_conf, {path, "../emqx_conf"}}, {emqx_utils, {path, "../emqx_utils"}} ]}. From ae3355880e06e40ea096f63741adc785981e2b16 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 3 Aug 2023 20:26:04 +0300 Subject: [PATCH 005/119] chore(auth): update emqx_authn tests --- .../src/emqx_authentication_config.erl | 3 ++- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 26 ++++++++++++------- .../test/emqx_authn_enable_flag_SUITE.erl | 16 +++++++----- .../emqx_authn/test/emqx_authn_http_SUITE.erl | 13 +++++----- .../test/emqx_authn_https_SUITE.erl | 13 +++++----- apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl | 19 +++++--------- .../test/emqx_authn_mnesia_SUITE.erl | 15 +++++------ .../test/emqx_authn_mongo_SUITE.erl | 15 +++++------ .../test/emqx_authn_mongo_tls_SUITE.erl | 15 +++++------ .../test/emqx_authn_mysql_SUITE.erl | 15 +++++------ .../test/emqx_authn_mysql_tls_SUITE.erl | 15 +++++------ .../test/emqx_authn_pgsql_SUITE.erl | 16 +++++------- .../test/emqx_authn_pgsql_tls_SUITE.erl | 13 +++++----- .../test/emqx_authn_redis_SUITE.erl | 15 +++++------ .../test/emqx_authn_redis_tls_SUITE.erl | 16 +++++------- .../test/emqx_authn_schema_SUITE.erl | 13 +++++----- ...emqx_enhanced_authn_scram_mnesia_SUITE.erl | 11 ++++---- 17 files changed, 123 insertions(+), 126 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authentication_config.erl b/apps/emqx_authn/src/emqx_authentication_config.erl index 13b9214de..cf8ba57b1 100644 --- a/apps/emqx_authn/src/emqx_authentication_config.erl +++ b/apps/emqx_authn/src/emqx_authentication_config.erl @@ -359,7 +359,8 @@ dir(ChainName, Config) when is_map(Config) -> chain_name([authentication]) -> ?GLOBAL; chain_name([listeners, Type, Name, authentication]) -> - binary_to_existing_atom(<<(atom_to_binary(Type))/binary, ":", (atom_to_binary(Name))/binary>>). + %% Type, Name atoms exist, so let 'Type:Name' exist too. + binary_to_atom(<<(atom_to_binary(Type))/binary, ":", (atom_to_binary(Name))/binary>>). merge_authenticators(OriginConf0, NewConf0) -> {OriginConf1, NewConf1} = diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index c0b3fe22f..5e740de15 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -23,6 +23,7 @@ -include("emqx_authn.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). -define(TCP_DEFAULT, 'tcp:default'). @@ -43,7 +44,6 @@ init_per_testcase(t_authenticator_fail, Config) -> meck:expect(emqx_authn_proto_v1, lookup_from_all_nodes, 3, [{error, {exception, badarg}}]), init_per_testcase(default, Config); init_per_testcase(_Case, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authn_test_lib:delete_authenticators( [?CONF_NS_ATOM], ?GLOBAL @@ -64,19 +64,27 @@ end_per_testcase(_, Config) -> Config. init_per_suite(Config) -> - emqx_config:erase(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY), - _ = application:load(emqx_conf), - ok = emqx_mgmt_api_test_util:init_suite( - [emqx_conf, emqx_authn] + Apps = emqx_cth_suite:start( + [ + emqx, + emqx_conf, + emqx_authn, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} + ], + #{ + work_dir => ?config(priv_dir, Config) + } ), - + _ = emqx_common_test_http:create_default_app(), ?AUTHN:delete_chain(?GLOBAL), {ok, Chains} = ?AUTHN:list_chains(), ?assertEqual(length(Chains), 0), - Config. + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_mgmt_api_test_util:end_suite([emqx_authn]), +end_per_suite(Config) -> + _ = emqx_common_test_http:delete_default_app(), + ok = emqx_cth_suite:stop(?config(apps, Config)), ok. %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl b/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl index cc2785b1e..ae2cc436e 100644 --- a/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl @@ -24,16 +24,19 @@ -define(PATH, [?CONF_NS_ATOM]). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:start_apps([emqx_conf, emqx_authn]), - Config. + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), + [{apps, Apps} | Config]. -end_per_suite(_) -> - emqx_common_test_helpers:stop_apps([emqx_authn, emqx_conf]), +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)), ok. init_per_testcase(_Case, Config) -> @@ -42,9 +45,10 @@ init_per_testcase(_Case, Config) -> <<"backend">> => <<"built_in_database">>, <<"user_id_type">> => <<"clientid">> }, - {ok, _} = emqx:update_config( + {ok, _} = emqx_conf:update( ?PATH, - {create_authenticator, ?GLOBAL, AuthnConfig} + {create_authenticator, ?GLOBAL, AuthnConfig}, + #{} ), {ok, _} = emqx_conf:update( [listeners, tcp, listener_authn_enabled], diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl index b08167a5b..dcd41a28d 100644 --- a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl @@ -65,18 +65,17 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - _ = application:load(emqx_conf), - emqx_common_test_helpers:start_apps([emqx_authn]), - application:ensure_all_started(cowboy), - Config. + Apps = emqx_cth_suite:start([cowboy, emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), + [{apps, Apps} | Config]. -end_per_suite(_) -> +end_per_suite(Config) -> emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL ), - emqx_common_test_helpers:stop_apps([emqx_authn]), - application:stop(cowboy), + ok = emqx_cth_suite:stop(?config(apps, Config)), ok. init_per_testcase(_Case, Config) -> diff --git a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl index c4315b69f..6fb8de294 100644 --- a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl @@ -39,18 +39,17 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - _ = application:load(emqx_conf), - emqx_common_test_helpers:start_apps([emqx_authn]), - application:ensure_all_started(cowboy), - Config. + Apps = emqx_cth_suite:start([cowboy, emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), + [{apps, Apps} | Config]. -end_per_suite(_) -> +end_per_suite(Config) -> emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL ), - emqx_common_test_helpers:stop_apps([emqx_authn]), - application:stop(cowboy), + ok = emqx_cth_suite:stop(?config(apps, Config)), ok. init_per_testcase(_Case, Config) -> diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index bd18367b6..75dfcbc6f 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -31,21 +31,14 @@ all() -> emqx_common_test_helpers:all(?MODULE). -init_per_testcase(_, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), - Config. - init_per_suite(Config) -> - _ = application:load(emqx_conf), - emqx_common_test_helpers:start_apps([emqx_conf, emqx_authn]), - application:ensure_all_started(emqx_resource), - application:ensure_all_started(emqx_connector), - Config. + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), + [{apps, Apps} | Config]. -end_per_suite(_) -> - application:stop(emqx_connector), - application:stop(emqx_resource), - emqx_common_test_helpers:stop_apps([emqx_authn]), +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)), ok. %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index 599eae92e..9781b8ca7 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -20,8 +20,7 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). - --include("emqx_authn.hrl"). +-include_lib("common_test/include/ct.hrl"). -define(AUTHN_ID, <<"mechanism:backend">>). @@ -29,16 +28,16 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - _ = application:load(emqx_conf), - emqx_common_test_helpers:start_apps([emqx_authn]), - Config. + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), + [{apps, Apps} | Config]. -end_per_suite(_) -> - emqx_common_test_helpers:stop_apps([emqx_authn]), +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)), ok. init_per_testcase(_Case, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), mria:clear_table(emqx_authn_mnesia), Config. diff --git a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl index 07933031e..9ea7f9eb2 100644 --- a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl @@ -33,7 +33,6 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_testcase(_TestCase, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -46,23 +45,23 @@ end_per_testcase(_TestCase, _Config) -> ok = mc_worker_api:disconnect(?MONGO_CLIENT). init_per_suite(Config) -> - _ = application:load(emqx_conf), case emqx_common_test_helpers:is_tcp_server_available(?MONGO_HOST, ?MONGO_DEFAULT_PORT) of true -> - ok = emqx_common_test_helpers:start_apps([emqx_authn]), - ok = start_apps([emqx_resource]), - Config; + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), + [{apps, Apps} | Config]; false -> {skip, no_mongo} end. -end_per_suite(_Config) -> +end_per_suite(Config) -> emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL ), - ok = stop_apps([emqx_resource]), - ok = emqx_common_test_helpers:stop_apps([emqx_authn]). + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. %%------------------------------------------------------------------------------ %% Tests diff --git a/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl index 34f906dd9..af550379b 100644 --- a/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl @@ -33,7 +33,6 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_testcase(_TestCase, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -42,23 +41,23 @@ init_per_testcase(_TestCase, Config) -> Config. init_per_suite(Config) -> - _ = application:load(emqx_conf), case emqx_common_test_helpers:is_tcp_server_available(?MONGO_HOST, ?MONGO_DEFAULT_PORT) of true -> - ok = emqx_common_test_helpers:start_apps([emqx_authn]), - ok = start_apps([emqx_resource]), - Config; + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), + [{apps, Apps} | Config]; false -> {skip, no_mongo} end. -end_per_suite(_Config) -> +end_per_suite(Config) -> emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL ), - ok = stop_apps([emqx_resource]), - ok = emqx_common_test_helpers:stop_apps([emqx_authn]). + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. %%------------------------------------------------------------------------------ %% Tests diff --git a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl index 914ce4dd1..2173b943b 100644 --- a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl @@ -37,7 +37,6 @@ groups() -> [{require_seeds, [], [t_authenticate, t_update, t_destroy]}]. init_per_testcase(_, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -54,11 +53,11 @@ end_per_group(require_seeds, Config) -> Config. init_per_suite(Config) -> - _ = application:load(emqx_conf), case emqx_common_test_helpers:is_tcp_server_available(?MYSQL_HOST, ?MYSQL_DEFAULT_PORT) of true -> - ok = emqx_common_test_helpers:start_apps([emqx_authn]), - ok = start_apps([emqx_resource]), + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), {ok, _} = emqx_resource:create_local( ?MYSQL_RESOURCE, ?RESOURCE_GROUP, @@ -66,19 +65,19 @@ init_per_suite(Config) -> mysql_config(), #{} ), - Config; + [{apps, Apps} | Config]; false -> {skip, no_mysql} end. -end_per_suite(_Config) -> +end_per_suite(Config) -> emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL ), ok = emqx_resource:remove_local(?MYSQL_RESOURCE), - ok = stop_apps([emqx_resource]), - ok = emqx_common_test_helpers:stop_apps([emqx_authn]). + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. %%------------------------------------------------------------------------------ %% Tests diff --git a/apps/emqx_authn/test/emqx_authn_mysql_tls_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_tls_SUITE.erl index 5d5a3f7ac..888ff5e6b 100644 --- a/apps/emqx_authn/test/emqx_authn_mysql_tls_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mysql_tls_SUITE.erl @@ -36,7 +36,6 @@ groups() -> []. init_per_testcase(_, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -45,23 +44,23 @@ init_per_testcase(_, Config) -> Config. init_per_suite(Config) -> - _ = application:load(emqx_conf), case emqx_common_test_helpers:is_tcp_server_available(?MYSQL_HOST, ?MYSQL_DEFAULT_PORT) of true -> - ok = emqx_common_test_helpers:start_apps([emqx_authn]), - ok = start_apps([emqx_resource]), - Config; + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), + [{apps, Apps} | Config]; false -> {skip, no_mysql_tls} end. -end_per_suite(_Config) -> +end_per_suite(Config) -> emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL ), - ok = stop_apps([emqx_resource]), - ok = emqx_common_test_helpers:stop_apps([emqx_authn]). + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. %%------------------------------------------------------------------------------ %% Tests diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl index 075ae5cb7..1c9f0f86b 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl @@ -23,7 +23,6 @@ -include_lib("emqx_authn/include/emqx_authn.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --include_lib("emqx/include/emqx_placeholder.hrl"). -define(PGSQL_HOST, "pgsql"). -define(PGSQL_RESOURCE, <<"emqx_authn_pgsql_SUITE">>). @@ -42,7 +41,6 @@ groups() -> [{require_seeds, [], [t_create, t_authenticate, t_update, t_destroy, t_is_superuser]}]. init_per_testcase(_, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -59,11 +57,11 @@ end_per_group(require_seeds, Config) -> Config. init_per_suite(Config) -> - _ = application:load(emqx_conf), case emqx_common_test_helpers:is_tcp_server_available(?PGSQL_HOST, ?PGSQL_DEFAULT_PORT) of true -> - ok = emqx_common_test_helpers:start_apps([emqx_authn]), - ok = start_apps([emqx_resource]), + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), {ok, _} = emqx_resource:create_local( ?PGSQL_RESOURCE, ?RESOURCE_GROUP, @@ -71,19 +69,19 @@ init_per_suite(Config) -> pgsql_config(), #{} ), - Config; + [{apps, Apps} | Config]; false -> {skip, no_pgsql} end. -end_per_suite(_Config) -> +end_per_suite(Config) -> emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL ), ok = emqx_resource:remove_local(?PGSQL_RESOURCE), - ok = stop_apps([emqx_resource]), - ok = emqx_common_test_helpers:stop_apps([emqx_authn]). + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. %%------------------------------------------------------------------------------ %% Tests diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_tls_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_tls_SUITE.erl index ae0a01572..4862572e6 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_tls_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_tls_SUITE.erl @@ -48,20 +48,21 @@ init_per_suite(Config) -> _ = application:load(emqx_conf), case emqx_common_test_helpers:is_tcp_server_available(?PGSQL_HOST, ?PGSQL_DEFAULT_PORT) of true -> - ok = emqx_common_test_helpers:start_apps([emqx_authn]), - ok = start_apps([emqx_resource]), - Config; + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), + [{apps, Apps} | Config]; false -> {skip, no_pgsql_tls} end. -end_per_suite(_Config) -> +end_per_suite(Config) -> emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL ), - ok = stop_apps([emqx_resource]), - ok = emqx_common_test_helpers:stop_apps([emqx_authn]). + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. %%------------------------------------------------------------------------------ %% Tests diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index 31602ecec..c8ae3d2a2 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -42,7 +42,6 @@ groups() -> [{require_seeds, [], [t_authenticate, t_update, t_destroy]}]. init_per_testcase(_, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -59,11 +58,11 @@ end_per_group(require_seeds, Config) -> Config. init_per_suite(Config) -> - _ = application:load(emqx_conf), case emqx_common_test_helpers:is_tcp_server_available(?REDIS_HOST, ?REDIS_DEFAULT_PORT) of true -> - ok = emqx_common_test_helpers:start_apps([emqx_authn]), - ok = start_apps([emqx_resource]), + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), {ok, _} = emqx_resource:create_local( ?REDIS_RESOURCE, ?RESOURCE_GROUP, @@ -71,19 +70,19 @@ init_per_suite(Config) -> redis_config(), #{} ), - Config; + [{apps, Apps} | Config]; false -> {skip, no_redis} end. -end_per_suite(_Config) -> +end_per_suite(Config) -> emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL ), ok = emqx_resource:remove_local(?REDIS_RESOURCE), - ok = stop_apps([emqx_resource]), - ok = emqx_common_test_helpers:stop_apps([emqx_authn]). + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. %%------------------------------------------------------------------------------ %% Tests diff --git a/apps/emqx_authn/test/emqx_authn_redis_tls_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_tls_SUITE.erl index 8df54ebce..291caed1b 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_tls_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_tls_SUITE.erl @@ -19,7 +19,6 @@ -compile(nowarn_export_all). -compile(export_all). --include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("emqx_authn/include/emqx_authn.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -36,7 +35,6 @@ groups() -> []. init_per_testcase(_, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -45,23 +43,23 @@ init_per_testcase(_, Config) -> Config. init_per_suite(Config) -> - _ = application:load(emqx_conf), case emqx_common_test_helpers:is_tcp_server_available(?REDIS_HOST, ?REDIS_TLS_PORT) of true -> - ok = emqx_common_test_helpers:start_apps([emqx_authn]), - ok = start_apps([emqx_resource]), - Config; + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), + [{apps, Apps} | Config]; false -> {skip, no_redis} end. -end_per_suite(_Config) -> +end_per_suite(Config) -> emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL ), - ok = stop_apps([emqx_resource]), - ok = emqx_common_test_helpers:stop_apps([emqx_authn]). + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. %%------------------------------------------------------------------------------ %% Tests diff --git a/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl b/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl index 3afb8e973..8266ade10 100644 --- a/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl @@ -4,6 +4,7 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). -include("emqx_authn.hrl"). @@ -11,16 +12,16 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - _ = application:load(emqx_conf), - emqx_common_test_helpers:start_apps([emqx_authn]), - Config. + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), + [{apps, Apps} | Config]. -end_per_suite(_) -> - emqx_common_test_helpers:stop_apps([emqx_authn]), +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)), ok. init_per_testcase(_Case, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), mria:clear_table(emqx_authn_mnesia), Config. diff --git a/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl index f52e895cc..baaf15175 100644 --- a/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl @@ -36,17 +36,18 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - _ = application:load(emqx_conf), - ok = emqx_common_test_helpers:start_apps([emqx_authn]), + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), IdleTimeout = emqx_config:get([mqtt, idle_timeout]), - [{idle_timeout, IdleTimeout} | Config]. + [{apps, Apps}, {idle_timeout, IdleTimeout} | Config]. end_per_suite(Config) -> ok = emqx_config:put([mqtt, idle_timeout], ?config(idle_timeout, Config)), - ok = emqx_common_test_helpers:stop_apps([emqx_authn]). + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. init_per_testcase(_Case, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), mria:clear_table(emqx_enhanced_authn_scram_mnesia), emqx_authn_test_lib:delete_authenticators( [authentication], From cd88dfbbb8c47b5279c620f66695d4b2f575a46f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 4 Aug 2023 18:01:24 +0300 Subject: [PATCH 006/119] chore(auth): update authn includes --- apps/emqx_authn/src/emqx_authentication_config.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authn/src/emqx_authentication_config.erl b/apps/emqx_authn/src/emqx_authentication_config.erl index cf8ba57b1..a00d0b555 100644 --- a/apps/emqx_authn/src/emqx_authentication_config.erl +++ b/apps/emqx_authn/src/emqx_authentication_config.erl @@ -39,7 +39,7 @@ -export_type([config/0]). --include("logger.hrl"). +-include_lib("emqx/include/logger.hrl"). -include("emqx_authentication.hrl"). -type parsed_config() :: #{ From 8a3e8ee528124229e4bb8e0ad4f73fe7d856bad3 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 4 Aug 2023 20:19:07 +0300 Subject: [PATCH 007/119] chore(auth): add config propagation tests for authn --- apps/emqx/src/emqx_config_handler.erl | 39 ++- apps/emqx/test/emqx_config_handler_SUITE.erl | 68 +++-- .../src/emqx_authentication_config.erl | 12 +- .../test/emqx_authn_listeners_SUITE.erl | 242 ++++++++++++++++++ 4 files changed, 316 insertions(+), 45 deletions(-) create mode 100644 apps/emqx_authn/test/emqx_authn_listeners_SUITE.erl diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index f68992e4e..e634112d7 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -63,7 +63,7 @@ -callback propagated_pre_config_update( [atom()], emqx_config:update_request(), emqx_config:raw_config() ) -> - ok | {error, term()}. + ok | {ok, emqx_config:update_request()} | {error, term()}. -callback post_config_update( [atom()], @@ -401,14 +401,14 @@ get_sub_config(_, _Conf) -> call_pre_config_update(Ctx) -> case call_proper_pre_config_update(Ctx) of - {ok, NewUpdateReq} -> + {ok, NewUpdateReq0} -> case propagate_pre_config_updates_to_subconf(Ctx#{ - update_req => NewUpdateReq + update_req => NewUpdateReq0 }) of - {ok, _} -> - {ok, NewUpdateReq}; + {ok, #{update_req := NewUpdateReq1}} -> + {ok, NewUpdateReq1}; {error, _} = Error -> Error end; @@ -471,12 +471,12 @@ propagate_pre_config_updates_to_subconf_wkey( Keys = propagate_keys(UpdateReq, OldRawConf), propagate_pre_config_updates_to_subconf_keys(Keys, Ctx). -propagate_pre_config_updates_to_subconf_keys([], #{update_req := UpdateReq}) -> - {ok, UpdateReq}; -propagate_pre_config_updates_to_subconf_keys([Key | Keys], Ctx) -> - case propagate_pre_config_updates_to_subconf_key(Key, Ctx) of - ok -> - propagate_pre_config_updates_to_subconf_keys(Keys, Ctx); +propagate_pre_config_updates_to_subconf_keys([], Ctx) -> + {ok, Ctx}; +propagate_pre_config_updates_to_subconf_keys([Key | Keys], Ctx0) -> + case propagate_pre_config_updates_to_subconf_key(Key, Ctx0) of + {ok, Ctx1} -> + propagate_pre_config_updates_to_subconf_keys(Keys, Ctx1); {error, _} = Error -> Error end. @@ -497,8 +497,10 @@ propagate_pre_config_updates_to_subconf_key( SubOldConf = get_sub_config(BinKey, OldRawConf), SubConfKeyPath = ConfKeyPath ++ [AtomKey], case {SubOldConf, SubUpdateReq} of + %% we have handler, but no relevant keys in both configs (new and old), + %% so we don't need to go further {undefined, undefined} -> - ok; + {ok, Ctx}; {_, _} -> case call_pre_config_update(Ctx#{ @@ -509,8 +511,17 @@ propagate_pre_config_updates_to_subconf_key( callback := propagated_pre_config_update }) of - {ok, _SubNewConf1} -> - ok; + {ok, SubNewConf1} -> + %% we update only if the new config is not to be removed + %% i.e. SubUpdateReq is not undefined + case SubUpdateReq of + undefined -> + {ok, Ctx}; + _ -> + {ok, Ctx#{ + update_req := maps:put(BinKey, SubNewConf1, UpdateReq) + }} + end; {error, _} = Error -> Error end diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index af2b4996c..ec24b54e6 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -224,8 +224,8 @@ t_callback_crash(_Config) -> ok = emqx_config_handler:remove_handler(CrashPath), ok. -t_pre_callback_error(_Config) -> - callback_error( +t_pre_assert_update_result(_Config) -> + assert_update_result( [sysmon, os, mem_check_interval], <<"100s">>, {error, {pre_config_update, ?MODULE, pre_config_update_error}} @@ -233,7 +233,7 @@ t_pre_callback_error(_Config) -> ok. t_post_update_error(_Config) -> - callback_error( + assert_update_result( [sysmon, os, sysmem_high_watermark], <<"60%">>, {error, {post_config_update, ?MODULE, post_config_update_error}} @@ -243,7 +243,7 @@ t_post_update_error(_Config) -> t_post_update_propagate_error_wkey(_Config) -> Conf0 = emqx_config:get_raw([sysmon]), Conf1 = emqx_utils_maps:deep_put([<<"os">>, <<"sysmem_high_watermark">>], Conf0, <<"60%">>), - callback_error( + assert_update_result( [ [sysmon, '?', sysmem_high_watermark], [sysmon] @@ -257,7 +257,7 @@ t_post_update_propagate_error_wkey(_Config) -> t_post_update_propagate_error_key(_Config) -> Conf0 = emqx_config:get_raw([sysmon]), Conf1 = emqx_utils_maps:deep_put([<<"os">>, <<"sysmem_high_watermark">>], Conf0, <<"60%">>), - callback_error( + assert_update_result( [ [sysmon, os, sysmem_high_watermark], [sysmon] @@ -271,7 +271,7 @@ t_post_update_propagate_error_key(_Config) -> t_pre_update_propagate_error_wkey(_Config) -> Conf0 = emqx_config:get_raw([sysmon]), Conf1 = emqx_utils_maps:deep_put([<<"os">>, <<"mem_check_interval">>], Conf0, <<"70s">>), - callback_error( + assert_update_result( [ [sysmon, '?', mem_check_interval], [sysmon] @@ -285,7 +285,7 @@ t_pre_update_propagate_error_wkey(_Config) -> t_pre_update_propagate_error_key(_Config) -> Conf0 = emqx_config:get_raw([sysmon]), Conf1 = emqx_utils_maps:deep_put([<<"os">>, <<"mem_check_interval">>], Conf0, <<"70s">>), - callback_error( + assert_update_result( [ [sysmon, os, mem_check_interval], [sysmon] @@ -296,6 +296,25 @@ t_pre_update_propagate_error_key(_Config) -> ), ok. +t_pre_update_propagate_key_rewrite(_Config) -> + Conf0 = emqx_config:get_raw([sysmon]), + Conf1 = emqx_utils_maps:deep_put([<<"os">>, <<"cpu_check_interval">>], Conf0, <<"333s">>), + with_update_result( + [ + [sysmon, '?', cpu_check_interval], + [sysmon] + ], + [sysmon], + Conf1, + fun(_, Result) -> + ?assertMatch( + {ok, #{config := #{os := #{cpu_check_interval := 444000}}}}, + Result + ) + end + ), + ok. + t_handler_root() -> %% Don't rely on default emqx_config_handler's merge behaviour. RootKey = [], @@ -352,6 +371,8 @@ pre_config_update([sysmon, os, sysmem_high_watermark], UpdateReq, _RawConf) -> pre_config_update([sysmon, os, mem_check_interval], _UpdateReq, _RawConf) -> {error, pre_config_update_error}. +propagated_pre_config_update([sysmon, os, cpu_check_interval], <<"333s">>, _RawConf) -> + {ok, <<"444s">>}; propagated_pre_config_update([sysmon, os, mem_check_interval], _UpdateReq, _RawConf) -> {error, pre_config_update_error}; propagated_pre_config_update(_ConfKeyPath, _UpdateReq, _RawConf) -> @@ -386,27 +407,32 @@ wait_for_new_pid() -> Pid end. -callback_error(FailedPath, Update, ExpectError) -> - callback_error([FailedPath], FailedPath, Update, ExpectError). +assert_update_result(FailedPath, Update, Expect) -> + assert_update_result([FailedPath], FailedPath, Update, Expect). -callback_error(Paths, UpdatePath, Update, ExpectError) -> +assert_update_result(Paths, UpdatePath, Update, Expect) -> + with_update_result(Paths, UpdatePath, Update, fun(Old, Result) -> + case Expect of + {error, {post_config_update, ?MODULE, post_config_update_error}} -> + ?assertMatch( + {error, {post_config_update, ?MODULE, {post_config_update_error, _}}}, Result + ); + _ -> + ?assertEqual(Expect, Result) + end, + New = emqx:get_raw_config(UpdatePath, undefined), + ?assertEqual(Old, New) + end). + +with_update_result(Paths, UpdatePath, Update, Fun) -> ok = lists:foreach( fun(Path) -> emqx_config_handler:add_handler(Path, ?MODULE) end, Paths ), Opts = #{rawconf_with_defaults => true}, Old = emqx:get_raw_config(UpdatePath, undefined), - Error = emqx:update_config(UpdatePath, Update, Opts), - case ExpectError of - {error, {post_config_update, ?MODULE, post_config_update_error}} -> - ?assertMatch( - {error, {post_config_update, ?MODULE, {post_config_update_error, _}}}, Error - ); - _ -> - ?assertEqual(ExpectError, Error) - end, - New = emqx:get_raw_config(UpdatePath, undefined), - ?assertEqual(Old, New), + Result = emqx:update_config(UpdatePath, Update, Opts), + _ = Fun(Old, Result), ok = lists:foreach( fun(Path) -> emqx_config_handler:remove_handler(Path) end, Paths diff --git a/apps/emqx_authn/src/emqx_authentication_config.erl b/apps/emqx_authn/src/emqx_authentication_config.erl index a00d0b555..431a2cf49 100644 --- a/apps/emqx_authn/src/emqx_authentication_config.erl +++ b/apps/emqx_authn/src/emqx_authentication_config.erl @@ -150,13 +150,12 @@ do_pre_config_update(Paths, NewConfig, _OldConfig) -> -spec propagated_pre_config_update(list(atom()), update_request(), emqx_config:raw_config()) -> ok | {error, term()}. propagated_pre_config_update(Paths, NewConfig, OldConfig) -> - {ok, _} = do_pre_config_update(Paths, NewConfig, OldConfig), - ok. + do_pre_config_update(Paths, NewConfig, OldConfig). -spec post_config_update( list(atom()), update_request(), - map() | list(), + map() | list() | undefined, emqx_config:raw_config(), emqx_config:app_envs() ) -> @@ -222,13 +221,6 @@ do_post_config_update(Paths, _UpdateReq, NewConfig0, OldConfig0, _AppEnvs) -> propagated_post_config_update(Paths, UpdateReq, NewConfig, OldConfig, AppEnvs) -> ok = post_config_update(Paths, UpdateReq, NewConfig, OldConfig, AppEnvs), - ChainName = chain_name(Paths), - ok = maybe_delete_chain(ChainName, NewConfig), - ok. - -maybe_delete_chain(ChainName, undefined) -> - ok = emqx_authentication:delete_chain(ChainName); -maybe_delete_chain(_ChainName, _NewConfig) -> ok. %% create new authenticators and update existing ones diff --git a/apps/emqx_authn/test/emqx_authn_listeners_SUITE.erl b/apps/emqx_authn/test/emqx_authn_listeners_SUITE.erl new file mode 100644 index 000000000..9708bf1bb --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_listeners_SUITE.erl @@ -0,0 +1,242 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_listeners_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx_authn.hrl"). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), + [{apps, Apps} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. + +init_per_testcase(_Case, Config) -> + Port = emqx_common_test_helpers:select_free_port(tcp), + [{port, Port} | Config]. + +end_per_testcase(_Case, _Config) -> + ok. + +t_create_update_delete(Config) -> + ListenerConf = listener_mqtt_tcp_conf(Config), + AuthnConfig0 = #{ + <<"mechanism">> => <<"password_based">>, + <<"backend">> => <<"built_in_database">>, + <<"user_id_type">> => <<"clientid">> + }, + %% Create + {ok, _} = emqx_conf:update( + [listeners], + #{ + <<"tcp">> => #{ + <<"listener0">> => ListenerConf#{ + ?CONF_NS_BINARY => AuthnConfig0 + } + } + }, + #{} + ), + ?assertMatch( + {ok, [ + #{ + authenticators := [ + #{ + id := <<"password_based:built_in_database">>, + state := #{ + user_id_type := clientid + } + } + ], + name := 'tcp:listener0' + } + ]}, + emqx_authentication:list_chains() + ), + + %% Drop old, create new + {ok, _} = emqx_conf:update( + [listeners], + #{ + <<"tcp">> => #{ + <<"listener1">> => ListenerConf#{ + ?CONF_NS_BINARY => AuthnConfig0 + } + } + }, + #{} + ), + ?assertMatch( + {ok, [ + #{ + authenticators := [ + #{ + id := <<"password_based:built_in_database">>, + state := #{ + user_id_type := clientid + } + } + ], + name := 'tcp:listener1' + } + ]}, + emqx_authentication:list_chains() + ), + + %% Update + {ok, _} = emqx_conf:update( + [listeners], + #{ + <<"tcp">> => #{ + <<"listener1">> => ListenerConf#{ + ?CONF_NS_BINARY => AuthnConfig0#{<<"user_id_type">> => <<"username">>} + } + } + }, + #{} + ), + ?assertMatch( + {ok, [ + #{ + authenticators := [ + #{ + id := <<"password_based:built_in_database">>, + state := #{ + user_id_type := username + } + } + ], + name := 'tcp:listener1' + } + ]}, + emqx_authentication:list_chains() + ), + + %% Update by listener path + {ok, _} = emqx_conf:update( + [listeners, tcp, listener1], + {update, ListenerConf#{ + ?CONF_NS_BINARY => AuthnConfig0#{<<"user_id_type">> => <<"clientid">>} + }}, + #{} + ), + ?assertMatch( + {ok, [ + #{ + authenticators := [ + #{ + id := <<"password_based:built_in_database">>, + state := #{ + user_id_type := clientid + } + } + ], + name := 'tcp:listener1' + } + ]}, + emqx_authentication:list_chains() + ), + + %% Delete + {ok, _} = emqx_conf:tombstone( + [listeners, tcp, listener1], + #{} + ), + ?assertMatch( + {ok, []}, + emqx_authentication:list_chains() + ). + +t_convert_certs(Config) -> + ListenerConf = listener_mqtt_tcp_conf(Config), + AuthnConfig0 = #{ + <<"mechanism">> => <<"password_based">>, + <<"password_hash_algorithm">> => #{ + <<"name">> => <<"plain">>, + <<"salt_position">> => <<"suffix">> + }, + <<"enable">> => <<"true">>, + + <<"backend">> => <<"redis">>, + <<"cmd">> => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>, + <<"database">> => <<"1">>, + <<"password">> => <<"public">>, + <<"server">> => <<"127.0.0.1:55555">>, + <<"redis_type">> => <<"single">>, + <<"ssl">> => #{ + <<"enable">> => true, + <<"cacertfile">> => some_pem(), + <<"certfile">> => some_pem(), + <<"keyfile">> => some_pem() + } + }, + {ok, _} = emqx_conf:update( + [listeners], + #{ + <<"tcp">> => #{ + <<"listener0">> => ListenerConf#{ + ?CONF_NS_BINARY => AuthnConfig0 + } + } + }, + #{} + ), + lists:foreach( + fun(Key) -> + [#{ssl := #{Key := FilePath}}] = emqx_config:get([ + listeners, tcp, listener0, authentication + ]), + ?assert(filelib:is_regular(FilePath)) + end, + [cacertfile, certfile, keyfile] + ). + +%%-------------------------------------------------------------------- +%% Helper Functions +%%-------------------------------------------------------------------- + +listener_mqtt_tcp_conf(Config) -> + Port = ?config(port, Config), + PortS = integer_to_binary(Port), + #{ + <<"acceptors">> => 16, + <<"access_rules">> => [<<"allow all">>], + <<"bind">> => <<"0.0.0.0:", PortS/binary>>, + <<"max_connections">> => 1024000, + <<"mountpoint">> => <<>>, + <<"proxy_protocol">> => false, + <<"proxy_protocol_timeout">> => <<"3s">>, + <<"enable_authn">> => true + }. + +some_pem() -> + Dir = code:lib_dir(emqx_authn, test), + Path = filename:join([Dir, "data", "private_key.pem"]), + {ok, Pem} = file:read_file(Path), + Pem. From f00f4568dc2ac5e0a20b9ca3a9942daa78dd69f2 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 7 Aug 2023 11:38:21 +0300 Subject: [PATCH 008/119] chore(auth): simplify schema injection --- apps/emqx/src/emqx_schema.erl | 4 ++-- apps/emqx/src/emqx_schema_hooks.erl | 22 +++++++--------------- apps/emqx_authn/src/emqx_authn_schema.erl | 4 ++-- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 3f4507a3c..40654a84b 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -222,7 +222,7 @@ roots(high) -> ref(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME), #{importance => ?IMPORTANCE_HIDDEN} )} - ] ++ emqx_schema_hooks:injection_point('roots.high'); + ] ++ emqx_schema_hooks:injection_point('authentication'); roots(medium) -> [ {"broker", @@ -1749,7 +1749,7 @@ mqtt_listener(Bind) -> default => <<"3s">> } )} - ] ++ emqx_schema_hooks:injection_point('mqtt.listener'). + ] ++ emqx_schema_hooks:injection_point('listeners.authentication'). base_listener(Bind) -> [ diff --git a/apps/emqx/src/emqx_schema_hooks.erl b/apps/emqx/src/emqx_schema_hooks.erl index 59fb1588b..fd80db635 100644 --- a/apps/emqx/src/emqx_schema_hooks.erl +++ b/apps/emqx/src/emqx_schema_hooks.erl @@ -28,9 +28,7 @@ -define(MODULE_PT_KEY(MOD_NAME), {?MODULE, mod, MOD_NAME}). -export([ - inject_fields/3, injection_point/1, - inject_fields_from_mod/1 ]). @@ -45,17 +43,7 @@ %%-------------------------------------------------------------------- injection_point(PointName) -> - InjectedFields = persistent_term:get(?HOOKPOINT_PT_KEY(PointName), #{}), - lists:concat(maps:values(InjectedFields)). - -inject_fields(PointName, Name, Fields) -> - Key = ?HOOKPOINT_PT_KEY(PointName), - InjectedFields = persistent_term:get(Key, #{}), - persistent_term:put(Key, InjectedFields#{Name => Fields}). - -%%-------------------------------------------------------------------- -%% Internal API -%%-------------------------------------------------------------------- + persistent_term:get(?HOOKPOINT_PT_KEY(PointName), []). inject_fields_from_mod(Module) -> case persistent_term:get(?MODULE_PT_KEY(Module), false) of @@ -105,10 +93,14 @@ do_inject_fields_from_mod(Module) -> ok end. -do_inject_fields_from_mod(Module, HookFields) -> +do_inject_fields_from_mod(_Module, HookFields) -> maps:foreach( fun(PointName, Fields) -> - inject_fields(PointName, Module, Fields) + inject_fields(PointName, Fields) end, HookFields ). + +inject_fields(PointName, Fields) -> + Key = ?HOOKPOINT_PT_KEY(PointName), + persistent_term:put(Key, Fields). diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index 2a12899aa..f77158f0c 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -42,8 +42,8 @@ roots() -> []. injected_fields() -> #{ - 'roots.high' => global_auth_fields(), - 'mqtt.listener' => mqtt_listener_auth_fields() + 'authentication' => global_auth_fields(), + 'listeners.authentication' => mqtt_listener_auth_fields() }. tags() -> From abcdf18ca4500bb8bfdeca2dded8b67b5c0563b2 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 7 Aug 2023 13:47:58 +0300 Subject: [PATCH 009/119] chore(auth): update ldap authn test --- apps/emqx_ldap/test/emqx_ldap_authn_SUITE.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx_ldap/test/emqx_ldap_authn_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_authn_SUITE.erl index d3b7a90f4..5167d2267 100644 --- a/apps/emqx_ldap/test/emqx_ldap_authn_SUITE.erl +++ b/apps/emqx_ldap/test/emqx_ldap_authn_SUITE.erl @@ -8,6 +8,7 @@ -include_lib("emqx_authn/include/emqx_authn.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). -define(LDAP_HOST, "ldap"). -define(LDAP_DEFAULT_PORT, 389). @@ -20,7 +21,6 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_testcase(_, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -32,8 +32,9 @@ init_per_suite(Config) -> _ = application:load(emqx_conf), case emqx_common_test_helpers:is_tcp_server_available(?LDAP_HOST, ?LDAP_DEFAULT_PORT) of true -> - ok = emqx_common_test_helpers:start_apps([emqx_authn]), - ok = start_apps([emqx_resource]), + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{ + work_dir => ?config(priv_dir, Config) + }), {ok, _} = emqx_resource:create_local( ?LDAP_RESOURCE, ?RESOURCE_GROUP, @@ -41,19 +42,18 @@ init_per_suite(Config) -> ldap_config(), #{} ), - Config; + [{apps, Apps} | Config]; false -> {skip, no_ldap} end. -end_per_suite(_Config) -> +end_per_suite(Config) -> emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL ), ok = emqx_resource:remove_local(?LDAP_RESOURCE), - ok = stop_apps([emqx_resource]), - ok = emqx_common_test_helpers:stop_apps([emqx_authn]). + ok = emqx_cth_suite:stop(?config(apps, Config)). %%------------------------------------------------------------------------------ %% Tests From 67e06b317183df7c296b18b91501f5ec428a2736 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 7 Aug 2023 19:29:50 +0300 Subject: [PATCH 010/119] chore(auth): make schema injection be more universal --- apps/emqx/src/emqx_schema.erl | 4 +- apps/emqx/src/emqx_schema_hooks.erl | 74 +++++++++++++---------- apps/emqx_authn/src/emqx_authn_schema.erl | 4 +- apps/emqx_conf/src/emqx_conf_schema.erl | 8 +-- 4 files changed, 48 insertions(+), 42 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 40654a84b..3f4507a3c 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -222,7 +222,7 @@ roots(high) -> ref(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME), #{importance => ?IMPORTANCE_HIDDEN} )} - ] ++ emqx_schema_hooks:injection_point('authentication'); + ] ++ emqx_schema_hooks:injection_point('roots.high'); roots(medium) -> [ {"broker", @@ -1749,7 +1749,7 @@ mqtt_listener(Bind) -> default => <<"3s">> } )} - ] ++ emqx_schema_hooks:injection_point('listeners.authentication'). + ] ++ emqx_schema_hooks:injection_point('mqtt.listener'). base_listener(Bind) -> [ diff --git a/apps/emqx/src/emqx_schema_hooks.erl b/apps/emqx/src/emqx_schema_hooks.erl index fd80db635..bd50a069f 100644 --- a/apps/emqx/src/emqx_schema_hooks.erl +++ b/apps/emqx/src/emqx_schema_hooks.erl @@ -25,11 +25,12 @@ -optional_callbacks([injected_fields/0]). -define(HOOKPOINT_PT_KEY(POINT_NAME), {?MODULE, fields, POINT_NAME}). --define(MODULE_PT_KEY(MOD_NAME), {?MODULE, mod, MOD_NAME}). -export([ injection_point/1, - inject_fields_from_mod/1 + any_injections/1, + inject_fields/2, + inject_from_modules/1 ]). %% for tests @@ -45,22 +46,15 @@ injection_point(PointName) -> persistent_term:get(?HOOKPOINT_PT_KEY(PointName), []). -inject_fields_from_mod(Module) -> - case persistent_term:get(?MODULE_PT_KEY(Module), false) of - false -> - persistent_term:put(?MODULE_PT_KEY(Module), true), - do_inject_fields_from_mod(Module); - true -> - ok - end. +inject_fields(PointName, Fields) -> + Key = ?HOOKPOINT_PT_KEY(PointName), + persistent_term:put(Key, Fields). erase_injections() -> lists:foreach( fun ({?HOOKPOINT_PT_KEY(_) = Key, _}) -> persistent_term:erase(Key); - ({?MODULE_PT_KEY(_) = Key, _}) -> - persistent_term:erase(Key); (_) -> ok end, @@ -72,35 +66,53 @@ any_injections() -> fun ({?HOOKPOINT_PT_KEY(_), _}) -> true; - ({?MODULE_PT_KEY(_), _}) -> - true; (_) -> false end, persistent_term:get() ). +any_injections(PointName) -> + persistent_term:get(?HOOKPOINT_PT_KEY(PointName), undefined) =/= undefined. + +inject_from_modules(Modules) -> + Injections = + lists:foldl( + fun append_module_injections/2, + #{}, + Modules + ), + ok = inject_fields(maps:to_list(Injections)). + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- -do_inject_fields_from_mod(Module) -> - _ = Module:module_info(), - case erlang:function_exported(Module, injected_fields, 0) of - true -> - do_inject_fields_from_mod(Module, Module:injected_fields()); - false -> - ok - end. - -do_inject_fields_from_mod(_Module, HookFields) -> - maps:foreach( - fun(PointName, Fields) -> - inject_fields(PointName, Fields) +append_module_injections(Module, AllInjections) when is_atom(Module) -> + append_module_injections(Module:injected_fields(), AllInjections); +append_module_injections(ModuleInjections, AllInjections) when is_map(ModuleInjections) -> + maps:fold( + fun(PointName, Fields, Acc) -> + maps:update_with( + PointName, + fun(Fields0) -> + Fields0 ++ Fields + end, + Fields, + Acc + ) end, - HookFields + AllInjections, + ModuleInjections ). -inject_fields(PointName, Fields) -> - Key = ?HOOKPOINT_PT_KEY(PointName), - persistent_term:put(Key, Fields). +inject_fields([]) -> + ok; +inject_fields([{PointName, Fields} | Rest]) -> + case emqx_schema_hooks:any_injections(PointName) of + true -> + inject_fields(Rest); + false -> + ok = emqx_schema_hooks:inject_fields(PointName, Fields), + inject_fields(Rest) + end. diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index f77158f0c..b0a68e702 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -42,8 +42,8 @@ roots() -> []. injected_fields() -> #{ - 'authentication' => global_auth_fields(), - 'listeners.authentication' => mqtt_listener_auth_fields() + 'mqtt.listener' => global_auth_fields(), + 'roots.high' => mqtt_listener_auth_fields() }. tags() -> diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 14cd4f17c..5b73b9e03 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -80,7 +80,7 @@ tags() -> [<<"EMQX">>]. roots() -> - ok = ensure_fields_injected(), + ok = emqx_schema_hooks:inject_from_modules(?INJECTING_CONFIGS), emqx_schema_high_prio_roots() ++ [ {"node", @@ -1434,9 +1434,3 @@ ensure_unicode_path(Path, _) when is_list(Path) -> Path; ensure_unicode_path(Path, _) -> throw({"not_string", Path}). - -ensure_fields_injected() -> - lists:foreach( - fun(Module) -> emqx_schema_hooks:inject_fields_from_mod(Module) end, - ?INJECTING_CONFIGS - ). From db31e5f0d946071c2cd560c7cdf4de49e7f6200c Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 8 Aug 2023 11:52:10 +0300 Subject: [PATCH 011/119] chore(auth): tidy up the code --- apps/emqx/include/emqx_hooks.hrl | 1 - apps/emqx/src/emqx_access_control.erl | 13 ++++------- apps/emqx/src/emqx_schema_hooks.erl | 22 +++++++++---------- .../include/emqx_authentication.hrl | 10 ++++----- apps/emqx_authn/src/emqx_authentication.erl | 8 +------ .../src/emqx_authentication_config.erl | 5 ++--- .../test/emqx_authentication_SUITE.erl | 2 +- 7 files changed, 24 insertions(+), 37 deletions(-) diff --git a/apps/emqx/include/emqx_hooks.hrl b/apps/emqx/include/emqx_hooks.hrl index dffc09fcd..2373b5928 100644 --- a/apps/emqx/include/emqx_hooks.hrl +++ b/apps/emqx/include/emqx_hooks.hrl @@ -29,7 +29,6 @@ -define(HP_RETAINER, 930). -define(HP_AUTO_SUB, 920). -define(HP_RULE_ENGINE, 900). - %% apps that can work with the republish action -define(HP_SLOW_SUB, 880). -define(HP_BRIDGE, 870). diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index a8ffadb44..82604710a 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -30,8 +30,8 @@ -compile(nowarn_export_all). -endif. --define(TRACE_RESULT(Label, Tag, Result, Reason), begin - ?TRACE(Label, Tag, #{ +-define(TRACE_RESULT(Label, Result, Reason), begin + ?TRACE(Label, ?AUTHN_TRACE_TAG, #{ result => (Result), reason => (Reason) }), @@ -115,18 +115,13 @@ authorize(ClientInfo, Action, Topic) -> -spec pre_hook_authenticate(emqx_types:clientinfo()) -> ok | continue | {error, not_authorized}. pre_hook_authenticate(#{enable_authn := false}) -> - ?TRACE_RESULT("pre_hook_authenticate", ?AUTHN_TRACE_TAG, ok, enable_authn_false); + ?TRACE_RESULT("pre_hook_authenticate", ok, enable_authn_false); pre_hook_authenticate(#{enable_authn := quick_deny_anonymous} = Credential) -> case is_username_defined(Credential) of true -> continue; false -> - ?TRACE_RESULT( - "pre_hook_authenticate", - ?AUTHN_TRACE_TAG, - {error, not_authorized}, - enable_authn_false - ) + ?TRACE_RESULT("pre_hook_authenticate", {error, not_authorized}, enable_authn_false) end; pre_hook_authenticate(_) -> continue. diff --git a/apps/emqx/src/emqx_schema_hooks.erl b/apps/emqx/src/emqx_schema_hooks.erl index bd50a069f..e704af6cc 100644 --- a/apps/emqx/src/emqx_schema_hooks.erl +++ b/apps/emqx/src/emqx_schema_hooks.erl @@ -24,12 +24,12 @@ }. -optional_callbacks([injected_fields/0]). +-export_type([hookpoint/0]). + -define(HOOKPOINT_PT_KEY(POINT_NAME), {?MODULE, fields, POINT_NAME}). -export([ injection_point/1, - any_injections/1, - inject_fields/2, inject_from_modules/1 ]). @@ -46,10 +46,6 @@ injection_point(PointName) -> persistent_term:get(?HOOKPOINT_PT_KEY(PointName), []). -inject_fields(PointName, Fields) -> - Key = ?HOOKPOINT_PT_KEY(PointName), - persistent_term:put(Key, Fields). - erase_injections() -> lists:foreach( fun @@ -72,9 +68,6 @@ any_injections() -> persistent_term:get() ). -any_injections(PointName) -> - persistent_term:get(?HOOKPOINT_PT_KEY(PointName), undefined) =/= undefined. - inject_from_modules(Modules) -> Injections = lists:foldl( @@ -109,10 +102,17 @@ append_module_injections(ModuleInjections, AllInjections) when is_map(ModuleInje inject_fields([]) -> ok; inject_fields([{PointName, Fields} | Rest]) -> - case emqx_schema_hooks:any_injections(PointName) of + case any_injections(PointName) of true -> inject_fields(Rest); false -> - ok = emqx_schema_hooks:inject_fields(PointName, Fields), + ok = inject_fields(PointName, Fields), inject_fields(Rest) end. + +inject_fields(PointName, Fields) -> + Key = ?HOOKPOINT_PT_KEY(PointName), + persistent_term:put(Key, Fields). + +any_injections(PointName) -> + persistent_term:get(?HOOKPOINT_PT_KEY(PointName), undefined) =/= undefined. diff --git a/apps/emqx_authn/include/emqx_authentication.hrl b/apps/emqx_authn/include/emqx_authentication.hrl index a367af291..c294b8d99 100644 --- a/apps/emqx_authn/include/emqx_authentication.hrl +++ b/apps/emqx_authn/include/emqx_authentication.hrl @@ -20,11 +20,6 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_access_control.hrl"). -%% config root name all auth providers have to agree on. --define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, "authentication"). --define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, authentication). --define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, <<"authentication">>). - -define(GLOBAL, 'mqtt:global'). -define(TRACE_AUTHN_PROVIDER(Msg), ?TRACE_AUTHN_PROVIDER(Msg, #{})). @@ -36,6 +31,11 @@ -define(TRACE_AUTHN(Msg, Meta), ?TRACE_AUTHN(debug, Msg, Meta)). -define(TRACE_AUTHN(Level, Msg, Meta), ?TRACE(Level, ?AUTHN_TRACE_TAG, Msg, Meta)). +%% config root name all auth providers have to agree on. +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, "authentication"). +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, authentication). +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, <<"authentication">>). + %% authentication move cmd -define(CMD_MOVE_FRONT, front). -define(CMD_MOVE_REAR, rear). diff --git a/apps/emqx_authn/src/emqx_authentication.erl b/apps/emqx_authn/src/emqx_authentication.erl index 5b49f5d28..8f055e049 100644 --- a/apps/emqx_authn/src/emqx_authentication.erl +++ b/apps/emqx_authn/src/emqx_authentication.erl @@ -60,7 +60,6 @@ register_providers/1, deregister_provider/1, deregister_providers/1, - providers/0, delete_chain/1, lookup_chain/1, list_chains/0, @@ -266,6 +265,7 @@ get_enabled(Authenticators) -> %%------------------------------------------------------------------------------ %% @doc Get all registered authentication providers. +-spec get_providers() -> #{authn_type() => module()}. get_providers() -> call(get_providers). @@ -332,10 +332,6 @@ deregister_providers(AuthNTypes) when is_list(AuthNTypes) -> deregister_provider(AuthNType) -> deregister_providers([AuthNType]). --spec providers() -> [{authn_type(), module()}]. -providers() -> - call(providers). - -spec delete_chain(chain_name()) -> ok | {error, term()}. delete_chain(Name) -> call({delete_chain, Name}). @@ -468,8 +464,6 @@ handle_call( end; handle_call({deregister_providers, AuthNTypes}, _From, #{providers := Providers} = State) -> reply(ok, State#{providers := maps:without(AuthNTypes, Providers)}); -handle_call(providers, _From, #{providers := Providers} = State) -> - reply(maps:to_list(Providers), State); handle_call({delete_chain, ChainName}, _From, State) -> UpdateFun = fun(Chain) -> {_MatchedIDs, NewChain} = do_delete_authenticators(fun(_) -> true end, Chain), diff --git a/apps/emqx_authn/src/emqx_authentication_config.erl b/apps/emqx_authn/src/emqx_authentication_config.erl index 431a2cf49..97d082da6 100644 --- a/apps/emqx_authn/src/emqx_authentication_config.erl +++ b/apps/emqx_authn/src/emqx_authentication_config.erl @@ -148,7 +148,7 @@ do_pre_config_update(Paths, NewConfig, _OldConfig) -> ]}. -spec propagated_pre_config_update(list(atom()), update_request(), emqx_config:raw_config()) -> - ok | {error, term()}. + {ok, map() | list()} | {error, term()}. propagated_pre_config_update(Paths, NewConfig, OldConfig) -> do_pre_config_update(Paths, NewConfig, OldConfig). @@ -217,8 +217,7 @@ do_post_config_update(Paths, _UpdateReq, NewConfig0, OldConfig0, _AppEnvs) -> emqx_config:raw_config(), emqx_config:app_envs() ) -> - ok | {ok, map()} | {error, term()}. - + ok. propagated_post_config_update(Paths, UpdateReq, NewConfig, OldConfig, AppEnvs) -> ok = post_config_update(Paths, UpdateReq, NewConfig, OldConfig, AppEnvs), ok. diff --git a/apps/emqx_authn/test/emqx_authentication_SUITE.erl b/apps/emqx_authn/test/emqx_authentication_SUITE.erl index dfeeb458e..a15f22c41 100644 --- a/apps/emqx_authn/test/emqx_authentication_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authentication_SUITE.erl @@ -591,5 +591,5 @@ deregister_providers() -> fun({Type, _Module}) -> ok = ?AUTHN:deregister_provider(Type) end, - lists:flatten([?AUTHN:providers()]) + maps:to_list(?AUTHN:get_providers()) ). From 0dae0400478bf2f034613fbe9a90a1d687c0377e Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 16 Aug 2023 20:26:43 +0300 Subject: [PATCH 012/119] chore(auth): restore injected schema key position --- apps/emqx/src/emqx_schema.erl | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 3f4507a3c..84695981a 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -214,15 +214,18 @@ roots(high) -> desc => ?DESC(zones), importance => ?IMPORTANCE_HIDDEN } - )}, - %% NOTE: authorization schema here is only to keep emqx app pure - %% the full schema for EMQX node is injected in emqx_conf_schema. - {?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME, - sc( - ref(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME), - #{importance => ?IMPORTANCE_HIDDEN} )} - ] ++ emqx_schema_hooks:injection_point('roots.high'); + ] ++ + emqx_schema_hooks:injection_point('roots.high') ++ + [ + %% NOTE: authorization schema here is only to keep emqx app pure + %% the full schema for EMQX node is injected in emqx_conf_schema. + {?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME, + sc( + ref(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME), + #{importance => ?IMPORTANCE_HIDDEN} + )} + ]; roots(medium) -> [ {"broker", From 3943830eae0f40ce7bf954a2dd940d15b4f0cf4b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 16 Aug 2023 20:27:36 +0300 Subject: [PATCH 013/119] chore(auth): remove unnecessary atom conversions --- apps/emqx/src/emqx_config_handler.erl | 24 +++++-- apps/emqx/test/emqx_config_handler_SUITE.erl | 8 ++- .../src/emqx_authentication_config.erl | 62 ++++++++++++------- apps/emqx_conf/src/emqx_conf.app.src | 2 +- 4 files changed, 65 insertions(+), 31 deletions(-) diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index e634112d7..a189fc9e5 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -61,7 +61,7 @@ -callback pre_config_update([atom()], emqx_config:update_request(), emqx_config:raw_config()) -> ok | {ok, emqx_config:update_request()} | {error, term()}. -callback propagated_pre_config_update( - [atom()], emqx_config:update_request(), emqx_config:raw_config() + [binary()], emqx_config:update_request(), emqx_config:raw_config() ) -> ok | {ok, emqx_config:update_request()} | {error, term()}. @@ -264,7 +264,8 @@ do_update_config([], Handlers, OldRawConf, UpdateReq, ConfKeyPath) -> old_raw_conf => OldRawConf, update_req => UpdateReq, conf_key_path => ConfKeyPath, - callback => pre_config_update + callback => pre_config_update, + is_propagated => false }); do_update_config( [ConfKey | SubConfKeyPath], @@ -391,6 +392,12 @@ get_sub_handlers(ConfKey, Handlers) when is_atom(ConfKey) -> case maps:find(ConfKey, Handlers) of error -> maps:get(?WKEY, Handlers, #{}); {ok, SubHandlers} -> SubHandlers + end; +get_sub_handlers(ConfKey, Handlers) when is_binary(ConfKey) -> + ConcreteHandlerKeys = maps:keys(Handlers) -- [?MOD, ?WKEY], + case lists:search(fun(K) -> bin(K) =:= ConfKey end, ConcreteHandlerKeys) of + {value, Key} -> maps:get(Key, Handlers); + false -> maps:get(?WKEY, Handlers, #{}) end. get_sub_config(ConfKey, Conf) when is_map(Conf) -> @@ -487,15 +494,19 @@ propagate_pre_config_updates_to_subconf_key( handlers := Handlers, old_raw_conf := OldRawConf, update_req := UpdateReq, - conf_key_path := ConfKeyPath + conf_key_path := ConfKeyPath, + is_propagated := IsPropagated } = Ctx ) -> - AtomKey = atom(Key), - SubHandlers = get_sub_handlers(AtomKey, Handlers), BinKey = bin(Key), + SubHandlers = get_sub_handlers(BinKey, Handlers), SubUpdateReq = get_sub_config(BinKey, UpdateReq), SubOldConf = get_sub_config(BinKey, OldRawConf), - SubConfKeyPath = ConfKeyPath ++ [AtomKey], + SubConfKeyPath = + case IsPropagated of + true -> ConfKeyPath ++ [BinKey]; + false -> bin_path(ConfKeyPath) ++ [BinKey] + end, case {SubOldConf, SubUpdateReq} of %% we have handler, but no relevant keys in both configs (new and old), %% so we don't need to go further @@ -508,6 +519,7 @@ propagate_pre_config_updates_to_subconf_key( old_raw_conf := SubOldConf, update_req := SubUpdateReq, conf_key_path := SubConfKeyPath, + is_propagated := true, callback := propagated_pre_config_update }) of diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index ec24b54e6..bb91bcbe4 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -371,9 +371,13 @@ pre_config_update([sysmon, os, sysmem_high_watermark], UpdateReq, _RawConf) -> pre_config_update([sysmon, os, mem_check_interval], _UpdateReq, _RawConf) -> {error, pre_config_update_error}. -propagated_pre_config_update([sysmon, os, cpu_check_interval], <<"333s">>, _RawConf) -> +propagated_pre_config_update( + [<<"sysmon">>, <<"os">>, <<"cpu_check_interval">>], <<"333s">>, _RawConf +) -> {ok, <<"444s">>}; -propagated_pre_config_update([sysmon, os, mem_check_interval], _UpdateReq, _RawConf) -> +propagated_pre_config_update( + [<<"sysmon">>, <<"os">>, <<"mem_check_interval">>], _UpdateReq, _RawConf +) -> {error, pre_config_update_error}; propagated_pre_config_update(_ConfKeyPath, _UpdateReq, _RawConf) -> ok. diff --git a/apps/emqx_authn/src/emqx_authentication_config.erl b/apps/emqx_authn/src/emqx_authentication_config.erl index 97d082da6..95140a0e8 100644 --- a/apps/emqx_authn/src/emqx_authentication_config.erl +++ b/apps/emqx_authn/src/emqx_authentication_config.erl @@ -67,8 +67,8 @@ -spec pre_config_update(list(atom()), update_request(), emqx_config:raw_config()) -> {ok, map() | list()} | {error, term()}. -pre_config_update(Paths, UpdateReq, OldConfig) -> - try do_pre_config_update(Paths, UpdateReq, to_list(OldConfig)) of +pre_config_update(ConfPath, UpdateReq, OldConfig) -> + try do_pre_config_update(ConfPath, UpdateReq, to_list(OldConfig)) of {error, Reason} -> {error, Reason}; {ok, NewConfig} -> {ok, NewConfig} catch @@ -132,25 +132,22 @@ do_pre_config_update(_, {move_authenticator, _ChainName, AuthenticatorID, Positi end end end; -do_pre_config_update(Paths, {merge_authenticators, NewConfig}, OldConfig) -> +do_pre_config_update(ConfPath, {merge_authenticators, NewConfig}, OldConfig) -> MergeConfig = merge_authenticators(OldConfig, NewConfig), - do_pre_config_update(Paths, MergeConfig, OldConfig); + do_pre_config_update(ConfPath, MergeConfig, OldConfig); do_pre_config_update(_, OldConfig, OldConfig) -> {ok, OldConfig}; -do_pre_config_update(Paths, NewConfig, _OldConfig) -> - ChainName = chain_name(Paths), - {ok, [ - begin - CertsDir = certs_dir(ChainName, New), - convert_certs(CertsDir, New) - end - || New <- to_list(NewConfig) - ]}. +do_pre_config_update(ConfPath, NewConfig, _OldConfig) -> + convert_certs_for_conf_path(ConfPath, NewConfig). --spec propagated_pre_config_update(list(atom()), update_request(), emqx_config:raw_config()) -> +%% @doc Handle listener config changes made at higher level. + +-spec propagated_pre_config_update(list(binary()), update_request(), emqx_config:raw_config()) -> {ok, map() | list()} | {error, term()}. -propagated_pre_config_update(Paths, NewConfig, OldConfig) -> - do_pre_config_update(Paths, NewConfig, OldConfig). +propagated_pre_config_update(_, OldConfig, OldConfig) -> + {ok, OldConfig}; +propagated_pre_config_update(ConfPath, NewConfig, _OldConfig) -> + convert_certs_for_conf_path(ConfPath, NewConfig). -spec post_config_update( list(atom()), @@ -160,8 +157,8 @@ propagated_pre_config_update(Paths, NewConfig, OldConfig) -> emqx_config:app_envs() ) -> ok | {ok, map()} | {error, term()}. -post_config_update(Paths, UpdateReq, NewConfig, OldConfig, AppEnvs) -> - do_post_config_update(Paths, UpdateReq, to_list(NewConfig), OldConfig, AppEnvs). +post_config_update(ConfPath, UpdateReq, NewConfig, OldConfig, AppEnvs) -> + do_post_config_update(ConfPath, UpdateReq, to_list(NewConfig), OldConfig, AppEnvs). do_post_config_update( _, {create_authenticator, ChainName, Config}, NewConfig, _OldConfig, _AppEnvs @@ -199,8 +196,8 @@ do_post_config_update( emqx_authentication:move_authenticator(ChainName, AuthenticatorID, Position); do_post_config_update(_, _UpdateReq, OldConfig, OldConfig, _AppEnvs) -> ok; -do_post_config_update(Paths, _UpdateReq, NewConfig0, OldConfig0, _AppEnvs) -> - ChainName = chain_name(Paths), +do_post_config_update(ConfPath, _UpdateReq, NewConfig0, OldConfig0, _AppEnvs) -> + ChainName = chain_name(ConfPath), OldConfig = to_list(OldConfig0), NewConfig = to_list(NewConfig0), OldIds = lists:map(fun authenticator_id/1, OldConfig), @@ -210,6 +207,8 @@ do_post_config_update(Paths, _UpdateReq, NewConfig0, OldConfig0, _AppEnvs) -> ok = emqx_authentication:reorder_authenticator(ChainName, NewIds), ok. +%% @doc Handle listener config changes made at higher level. + -spec propagated_post_config_update( list(atom()), update_request(), @@ -218,8 +217,8 @@ do_post_config_update(Paths, _UpdateReq, NewConfig0, OldConfig0, _AppEnvs) -> emqx_config:app_envs() ) -> ok. -propagated_post_config_update(Paths, UpdateReq, NewConfig, OldConfig, AppEnvs) -> - ok = post_config_update(Paths, UpdateReq, NewConfig, OldConfig, AppEnvs), +propagated_post_config_update(ConfPath, UpdateReq, NewConfig, OldConfig, AppEnvs) -> + ok = post_config_update(ConfPath, UpdateReq, NewConfig, OldConfig, AppEnvs), ok. %% create new authenticators and update existing ones @@ -257,6 +256,17 @@ to_list(M) when M =:= #{} -> []; to_list(M) when is_map(M) -> [M]; to_list(L) when is_list(L) -> L. +convert_certs_for_conf_path(ConfPath, NewConfig) -> + ChainName = chain_name_for_filepath(ConfPath), + CovertedConfs = lists:map( + fun(Conf) -> + CertsDir = certs_dir(ChainName, Conf), + convert_certs(CertsDir, Conf) + end, + to_list(NewConfig) + ), + {ok, CovertedConfs}. + convert_certs(CertsDir, NewConfig) -> NewSSL = maps:get(<<"ssl">>, NewConfig, undefined), case emqx_tls_lib:ensure_ssl_files(CertsDir, NewSSL) of @@ -353,6 +363,14 @@ chain_name([listeners, Type, Name, authentication]) -> %% Type, Name atoms exist, so let 'Type:Name' exist too. binary_to_atom(<<(atom_to_binary(Type))/binary, ":", (atom_to_binary(Name))/binary>>). +chain_name_for_filepath(Path) -> + do_chain_name_for_filepath([to_bin(Key) || Key <- Path]). + +do_chain_name_for_filepath([<<"authentication">>]) -> + to_bin(?GLOBAL); +do_chain_name_for_filepath([<<"listeners">>, Type, Name, <<"authentication">>]) -> + <<(to_bin(Type))/binary, ":", (to_bin(Name))/binary>>. + merge_authenticators(OriginConf0, NewConf0) -> {OriginConf1, NewConf1} = lists:foldl( diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index ab65c03c8..a4781e6fb 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.26"}, + {vsn, "0.1.27"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, From d17bcc94f6e2fe743a0407dcc3af9e03dab83ea6 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 16 Aug 2023 22:37:42 +0300 Subject: [PATCH 014/119] chore(auth): update emqx_gcp_device tests --- .../test/emqx_gcp_device_SUITE.erl | 14 +++++--- .../test/emqx_gcp_device_api_SUITE.erl | 34 ++++++++++--------- .../test/emqx_gcp_device_authn_SUITE.erl | 12 ++++--- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/apps/emqx_gcp_device/test/emqx_gcp_device_SUITE.erl b/apps/emqx_gcp_device/test/emqx_gcp_device_SUITE.erl index 5f286d629..4c8e89551 100644 --- a/apps/emqx_gcp_device/test/emqx_gcp_device_SUITE.erl +++ b/apps/emqx_gcp_device/test/emqx_gcp_device_SUITE.erl @@ -17,15 +17,19 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_authn, emqx_retainer, emqx_gcp_device]), - Config. + Apps = emqx_cth_suite:start( + [emqx, emqx_conf, emqx_authn, emqx_gcp_device, {emqx_retainer, "retainer {enable = true}"}], + #{ + work_dir => ?config(priv_dir, Config) + } + ), + [{apps, Apps} | Config]. end_per_suite(Config) -> - _ = emqx_common_test_helpers:stop_apps([emqx_authn, emqx_retainer, emqx_gcp_device]), - Config. + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. init_per_testcase(_TestCase, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL diff --git a/apps/emqx_gcp_device/test/emqx_gcp_device_api_SUITE.erl b/apps/emqx_gcp_device/test/emqx_gcp_device_api_SUITE.erl index 238f99445..4ed34344e 100644 --- a/apps/emqx_gcp_device/test/emqx_gcp_device_api_SUITE.erl +++ b/apps/emqx_gcp_device/test/emqx_gcp_device_api_SUITE.erl @@ -14,32 +14,34 @@ -include_lib("emqx/include/emqx.hrl"). -define(PATH, [authentication]). --define(BASE_CONF, << - "" - "\n" - "retainer {\n" - " enable = true\n" - "}" - "" ->>). all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_config:init_load(emqx_retainer_schema, ?BASE_CONF), - ok = emqx_common_test_helpers:start_apps([emqx_gcp_device, emqx_authn, emqx_conf, emqx_retainer]), - emqx_dashboard_api_test_helpers:set_default_config(), - emqx_mgmt_api_test_util:init_suite(), - Config. + Apps = emqx_cth_suite:start( + [ + emqx, + emqx_conf, + emqx_authn, + {emqx_retainer, "retainer {enable = true}"}, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}, + emqx_gcp_device + ], + #{ + work_dir => ?config(priv_dir, Config) + } + ), + _ = emqx_common_test_http:create_default_app(), + [{apps, Apps} | Config]. end_per_suite(Config) -> - emqx_mgmt_api_test_util:end_suite(), - _ = emqx_common_test_helpers:stop_apps([emqx_authn, emqx_retainer, emqx_gcp_device]), + _ = emqx_common_test_http:delete_default_app(), + ok = emqx_cth_suite:stop(?config(apps, Config)), Config. init_per_testcase(_TestCase, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL diff --git a/apps/emqx_gcp_device/test/emqx_gcp_device_authn_SUITE.erl b/apps/emqx_gcp_device/test/emqx_gcp_device_authn_SUITE.erl index 8c3f8e0fa..23e69f4c5 100644 --- a/apps/emqx_gcp_device/test/emqx_gcp_device_authn_SUITE.erl +++ b/apps/emqx_gcp_device/test/emqx_gcp_device_authn_SUITE.erl @@ -23,7 +23,9 @@ all() -> init_per_suite(Config0) -> ok = snabbkaffe:start_trace(), - emqx_common_test_helpers:start_apps([emqx_conf, emqx_authn, emqx_gcp_device]), + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn, emqx_gcp_device], #{ + work_dir => ?config(priv_dir, Config0) + }), ValidExpirationTime = erlang:system_time(second) + 3600, ValidJWT = generate_jws(ValidExpirationTime), ExpiredJWT = generate_jws(0), @@ -35,16 +37,16 @@ init_per_suite(Config0) -> {valid_jwt, ValidJWT}, {expired_jwt, ExpiredJWT}, {valid_client, ValidClient}, - {expired_client, ExpiredClient} + {expired_client, ExpiredClient}, + {apps, Apps} | Config0 ]. -end_per_suite(_) -> - _ = emqx_common_test_helpers:stop_apps([emqx_authn, emqx_gcp_device]), +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)), ok. init_per_testcase(_, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), Config. end_per_testcase(_Case, Config) -> From ebecbd154566196be6a2e588a47ec49fd98f9265 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 16 Aug 2023 13:42:01 -0300 Subject: [PATCH 015/119] fix(bridge): make dryrun health check timeout more malleable Fixes https://emqx.atlassian.net/browse/EMQX-10773 - Makes the timeout for probing a bridge more malleable to account for differences between each database. - Increases GCP PubSub Consumer default health check timeout to account for GCP slowness/throttling. --- .../src/emqx_bridge_gcp_pubsub.app.src | 2 +- apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl | 5 ++++- apps/emqx_resource/src/emqx_resource.app.src | 2 +- apps/emqx_resource/src/emqx_resource_manager.erl | 4 +++- apps/emqx_resource/src/schema/emqx_resource_schema.erl | 1 + changes/ee/fix-11461.en.md | 1 + 6 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 changes/ee/fix-11461.en.md diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src index 9faf65860..c7dcea5c0 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_gcp_pubsub, [ {description, "EMQX Enterprise GCP Pub/Sub Bridge"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl index b3792da71..6da1e3197 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl @@ -191,7 +191,10 @@ fields(consumer_topic_mapping) -> )} ]; fields("consumer_resource_opts") -> - ResourceFields = emqx_resource_schema:fields("creation_opts"), + ResourceFields = + emqx_resource_schema:create_opts( + [{health_check_interval, #{default => <<"30s">>}}] + ), SupportedFields = [ auto_restart_interval, health_check_interval, diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 283262e99..fdd760de0 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.21"}, + {vsn, "0.1.22"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 2e4822a2f..27936851e 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -179,7 +179,9 @@ create_dry_run(ResourceType, Config) -> false -> #{} end, ok = emqx_resource_manager_sup:ensure_child(ResId, <<"dry_run">>, ResourceType, Config, Opts), - case wait_for_ready(ResId, 5000) of + HealthCheckInterval = maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL), + Timeout = emqx_utils:clamp(HealthCheckInterval, 5_000, 60_000), + case wait_for_ready(ResId, Timeout) of ok -> remove(ResId); {error, Reason} -> diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index b98f50a98..dcf3414e6 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -47,6 +47,7 @@ fields("resource_opts") -> fields("creation_opts") -> create_opts([]). +-spec create_opts([{atom(), hocon_schema:field_schema_map()}]) -> [{atom(), hocon_schema:field()}]. create_opts(Overrides) -> override( [ diff --git a/changes/ee/fix-11461.en.md b/changes/ee/fix-11461.en.md new file mode 100644 index 000000000..011e3e31e --- /dev/null +++ b/changes/ee/fix-11461.en.md @@ -0,0 +1 @@ +Made the timeout for testing bridges connectivity follow more closely the configured health check timeout. From afffdbbaa1a1a66fd0b90b846e3de4ac511df9cf Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Tue, 15 Aug 2023 14:10:38 -0300 Subject: [PATCH 016/119] fix(kinesis): replace default payload template Fixes https://emqx.atlassian.net/browse/EMQX-10766 --- .../src/emqx_bridge_kinesis.erl | 2 +- ...mqx_bridge_kinesis_impl_producer_SUITE.erl | 44 ++++++++++++++++++- changes/ee/fix-11452.en.md | 1 + 3 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 changes/ee/fix-11452.en.md diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl index cb3cd3788..5ed60c8aa 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl @@ -83,7 +83,7 @@ fields(producer) -> sc( binary(), #{ - default => <<>>, + default => <<"${.}">>, desc => ?DESC("payload_template") } )}, diff --git a/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl b/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl index d0fe4a1b4..69b4a2d86 100644 --- a/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl @@ -228,13 +228,17 @@ create_bridge_http(Config, KinesisConfigOverrides) -> Res. create_bridge(Config) -> - create_bridge(Config, _KinesisConfigOverrides = #{}). + create_bridge(Config, #{}, []). create_bridge(Config, KinesisConfigOverrides) -> + create_bridge(Config, KinesisConfigOverrides, []). + +create_bridge(Config, KinesisConfigOverrides, Removes) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(kinesis_name, Config), KinesisConfig0 = ?config(kinesis_config, Config), - KinesisConfig = emqx_utils_maps:deep_merge(KinesisConfig0, KinesisConfigOverrides), + KinesisConfig1 = emqx_utils_maps:deep_merge(KinesisConfig0, KinesisConfigOverrides), + KinesisConfig = emqx_utils_maps:deep_remove(Removes, KinesisConfig1), ct:pal("creating bridge: ~p", [KinesisConfig]), Res = emqx_bridge:create(TypeBin, Name, KinesisConfig), ct:pal("bridge creation result: ~p", [Res]), @@ -862,3 +866,39 @@ t_access_denied(Config) -> end ), ok. + +t_empty_payload_template(Config) -> + ResourceId = ?config(resource_id, Config), + TelemetryTable = ?config(telemetry_table, Config), + Removes = [<<"payload_template">>], + ?assertMatch({ok, _}, create_bridge(Config, #{}, Removes)), + {ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config), + emqx_common_test_helpers:on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), + assert_empty_metrics(ResourceId), + ShardIt = get_shard_iterator(Config), + Payload = <<"payload">>, + Message = emqx_message:make(?TOPIC, Payload), + emqx:publish(Message), + %% to avoid test flakiness + wait_telemetry_event(TelemetryTable, success, ResourceId), + wait_until_gauge_is(queuing, 0, 500), + wait_until_gauge_is(inflight, 0, 500), + assert_metrics( + #{ + dropped => 0, + failed => 0, + inflight => 0, + matched => 1, + queuing => 0, + retried => 0, + success => 1 + }, + ResourceId + ), + Record = wait_record(Config, ShardIt, 100, 10), + Data = proplists:get_value(<<"Data">>, Record), + ?assertMatch( + #{<<"payload">> := <<"payload">>, <<"topic">> := ?TOPIC}, + emqx_utils_json:decode(Data, [return_maps]) + ), + ok. diff --git a/changes/ee/fix-11452.en.md b/changes/ee/fix-11452.en.md new file mode 100644 index 000000000..bd6080fce --- /dev/null +++ b/changes/ee/fix-11452.en.md @@ -0,0 +1 @@ +The default payload template for Kinesis was updated to store the entire message when no template is provided. From befc26055562edeea80a2ea91af76e76ed7d7ec3 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 17 Aug 2023 11:09:39 -0300 Subject: [PATCH 017/119] fix(gcp_consumer): remove unused resource_opts fields Fixes https://emqx.atlassian.net/browse/EMQX-10789 --- apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl index d1e827d84..ca7499a7b 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl @@ -209,11 +209,8 @@ fields(consumer_topic_mapping) -> fields("consumer_resource_opts") -> ResourceFields = emqx_resource_schema:fields("creation_opts"), SupportedFields = [ - auto_restart_interval, health_check_interval, - request_ttl, - resume_interval, - worker_pool_size + request_ttl ], lists:filter( fun({Field, _Sc}) -> lists:member(Field, SupportedFields) end, From 70ebb0b7516495f3a47cf2d9379729f50f2a6e7e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 18 Aug 2023 10:38:44 -0300 Subject: [PATCH 018/119] docs: change password label for azure event hub bridge Changed to use more specific AEH nomenclature. --- rel/i18n/emqx_bridge_azure_event_hub.hocon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/i18n/emqx_bridge_azure_event_hub.hocon b/rel/i18n/emqx_bridge_azure_event_hub.hocon index e6b3172b9..a0ccb0f2f 100644 --- a/rel/i18n/emqx_bridge_azure_event_hub.hocon +++ b/rel/i18n/emqx_bridge_azure_event_hub.hocon @@ -291,10 +291,10 @@ auth_username_password.label: """Username/password Auth""" auth_sasl_password.desc: -"""The password for connecting to Azure Event Hub. Should be the "connection string-primary key" of a Namespace shared access policy.""" +"""The Connection String for connecting to Azure Event Hub. Should be the "connection string-primary key" of a Namespace shared access policy.""" auth_sasl_password.label: -"""Password""" +"""Connection String""" producer_kafka_opts.desc: """Azure Event Hub producer configs.""" From 66dec69d09254b98bd2f62201eeca7f7a289dc50 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 18 Aug 2023 13:30:28 -0300 Subject: [PATCH 019/119] fix(rule_engine): capture function_clause errors Fixes https://emqx.atlassian.net/browse/EMQX-10798 --- .../src/emqx_rule_runtime.erl | 19 ++++-- .../test/emqx_rule_engine_SUITE.erl | 60 +++++++++++++++++++ changes/ce/fix-11480.en.md | 1 + 3 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 changes/ce/fix-11480.en.md diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index f047e2047..74396dbc8 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -493,11 +493,20 @@ apply_func(Other, _, _) -> }). do_apply_func(Module, Name, Args, Columns) -> - case erlang:apply(Module, Name, Args) of - Func when is_function(Func) -> - erlang:apply(Func, [Columns]); - Result -> - Result + try + case erlang:apply(Module, Name, Args) of + Func when is_function(Func) -> + erlang:apply(Func, [Columns]); + Result -> + Result + end + catch + error:function_clause -> + ?RAISE_BAD_SQL(#{ + reason => bad_sql_function_argument, + arguments => Args, + function_name => Name + }) end. add_metadata(Columns, Metadata) when is_map(Columns), is_map(Metadata) -> diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 8c3bd0ebb..ae4bf43f6 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -293,6 +293,66 @@ t_kv_store(_) -> emqx_rule_funcs:kv_store_del(<<"abc">>), undefined = emqx_rule_funcs:kv_store_get(<<"abc">>). +t_function_clause_errors(_Config) -> + SQL0 = <<"select upper(xxxx) from \"t/a\"">>, + Payload = <<"{}">>, + ?assertMatch( + {error, + {select_and_transform_error, + {throw, + #{ + arguments := [undefined], + reason := bad_sql_function_argument, + function_name := upper + }, + _Stack}}}, + emqx_rule_sqltester:test( + #{ + sql => SQL0, + context => #{payload => Payload, topic => <<"t/a">>} + } + ) + ), + SQL1 = <<"foreach xs as x do upper(xxxx) from \"t/a\"">>, + ?assertMatch( + {error, { + {doeach_error, + {throw, + #{ + arguments := [undefined], + reason := bad_sql_function_argument, + function_name := upper + }, + _Stack0}}, + _Stack1 + }}, + emqx_rule_sqltester:test( + #{ + sql => SQL1, + context => #{payload => Payload, xs => [1, 2, 3], topic => <<"t/a">>} + } + ) + ), + SQL2 = <<"foreach upper(xxxx) as x from \"t/a\"">>, + ?assertMatch( + {error, + {select_and_collect_error, + {throw, + #{ + arguments := [undefined], + reason := bad_sql_function_argument, + function_name := upper + }, + _Stack}}}, + emqx_rule_sqltester:test( + #{ + sql => SQL2, + context => #{payload => Payload, topic => <<"t/a">>} + } + ) + ), + ok. + %%------------------------------------------------------------------------------ %% Test cases for rule registry %%------------------------------------------------------------------------------ diff --git a/changes/ce/fix-11480.en.md b/changes/ce/fix-11480.en.md new file mode 100644 index 000000000..f3440f59e --- /dev/null +++ b/changes/ce/fix-11480.en.md @@ -0,0 +1 @@ +Return more user-friendly messages when rule functions are fed bad arguments. From fc3720627bc60d7b077ef4ba910a729fdf7ccbad Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 17 Aug 2023 19:04:19 +0300 Subject: [PATCH 020/119] chore(ci): fix zookeeper version for hstreamdb --- .ci/docker-compose-file/.env | 2 ++ .ci/docker-compose-file/docker-compose-hstreamdb.yaml | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index e99a6d13f..b7033caae 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -10,6 +10,8 @@ CASSANDRA_TAG=3.11.6 MINIO_TAG=RELEASE.2023-03-20T20-16-18Z OPENTS_TAG=9aa7f88 KINESIS_TAG=2.1 +HSTREAMDB_TAG=v0.15.0 +HSTREAMDB_ZK_TAG=3.8.1 MS_IMAGE_ADDR=mcr.microsoft.com/mssql/server SQLSERVER_TAG=2019-CU19-ubuntu-20.04 diff --git a/.ci/docker-compose-file/docker-compose-hstreamdb.yaml b/.ci/docker-compose-file/docker-compose-hstreamdb.yaml index bf367e408..d42fd9fa2 100644 --- a/.ci/docker-compose-file/docker-compose-hstreamdb.yaml +++ b/.ci/docker-compose-file/docker-compose-hstreamdb.yaml @@ -2,7 +2,7 @@ version: "3.5" services: hserver: - image: hstreamdb/hstream:v0.15.0 + image: hstreamdb/hstream:${HSTREAMDB_TAG} container_name: hstreamdb depends_on: zookeeper: @@ -39,7 +39,7 @@ services: --io-tasks-network emqx_bridge hstore: - image: hstreamdb/hstream:v0.15.0 + image: hstreamdb/hstream:${HSTREAMDB_TAG} networks: - emqx_bridge volumes: @@ -65,7 +65,7 @@ services: start_period: 60s zookeeper: - image: zookeeper + image: zookeeper:${HSTREAMDB_ZK_TAG} expose: - 2181 networks: From 2c458b62f5e9761dbef09ab17faff5e071a0ab1e Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Mon, 7 Aug 2023 11:27:30 -0300 Subject: [PATCH 021/119] feat(redis): add support for `username` on AUTH command Fixes https://emqx.atlassian.net/browse/EMQX-9911 --- .../docker-compose-redis-single-tcp.yaml | 7 ++-- .../docker-compose-redis-single-tls.yaml | 12 ++----- .../redis/cluster-tcp/redis.conf | 3 +- .../redis/cluster-tcp/users.acl | 2 ++ .../redis/cluster-tls/redis.conf | 3 +- .../redis/cluster-tls/users.acl | 2 ++ .../redis/sentinel-tcp/master.conf | 2 +- .../redis/sentinel-tcp/slave.conf | 3 +- .../redis/sentinel-tcp/users.acl | 2 ++ .../redis/sentinel-tls/master.conf | 2 +- .../redis/sentinel-tls/slave.conf | 3 +- .../redis/sentinel-tls/users.acl | 2 ++ .../redis/single-tcp/redis.conf | 3 ++ .../redis/single-tcp/users.acl | 2 ++ .../redis/single-tls/redis.conf | 9 +++++ .../redis/single-tls/users.acl | 2 ++ .../src/emqx_bridge_redis.app.src | 2 +- .../test/emqx_bridge_redis_SUITE.erl | 34 +++++++++++++++++++ apps/emqx_redis/rebar.config | 2 +- apps/emqx_redis/src/emqx_redis.app.src | 2 +- apps/emqx_redis/src/emqx_redis.erl | 2 ++ apps/emqx_redis/test/emqx_redis_SUITE.erl | 28 ++++++++++++++- changes/ce/feat-11469.en.md | 1 + 23 files changed, 106 insertions(+), 24 deletions(-) create mode 100644 .ci/docker-compose-file/redis/cluster-tcp/users.acl create mode 100644 .ci/docker-compose-file/redis/cluster-tls/users.acl create mode 100644 .ci/docker-compose-file/redis/sentinel-tcp/users.acl create mode 100644 .ci/docker-compose-file/redis/sentinel-tls/users.acl create mode 100644 .ci/docker-compose-file/redis/single-tcp/redis.conf create mode 100644 .ci/docker-compose-file/redis/single-tcp/users.acl create mode 100644 .ci/docker-compose-file/redis/single-tls/redis.conf create mode 100644 .ci/docker-compose-file/redis/single-tls/users.acl create mode 100644 changes/ce/feat-11469.en.md diff --git a/.ci/docker-compose-file/docker-compose-redis-single-tcp.yaml b/.ci/docker-compose-file/docker-compose-redis-single-tcp.yaml index 6706fe84f..ec7283219 100644 --- a/.ci/docker-compose-file/docker-compose-redis-single-tcp.yaml +++ b/.ci/docker-compose-file/docker-compose-redis-single-tcp.yaml @@ -4,12 +4,11 @@ services: redis_server: container_name: redis image: redis:${REDIS_TAG} + volumes: + - ./redis/single-tcp:/usr/local/etc/redis/ ports: - "6379:6379" - command: - - redis-server - - "--bind 0.0.0.0 ::" - - --requirepass public + command: redis-server /usr/local/etc/redis/redis.conf restart: always networks: - emqx_bridge diff --git a/.ci/docker-compose-file/docker-compose-redis-single-tls.yaml b/.ci/docker-compose-file/docker-compose-redis-single-tls.yaml index 8f59e7a9e..2ea36affd 100644 --- a/.ci/docker-compose-file/docker-compose-redis-single-tls.yaml +++ b/.ci/docker-compose-file/docker-compose-redis-single-tls.yaml @@ -8,18 +8,10 @@ services: - ./certs/server.crt:/etc/certs/redis.crt - ./certs/server.key:/etc/certs/redis.key - ./certs/ca.crt:/etc/certs/ca.crt + - ./redis/single-tls:/usr/local/etc/redis ports: - "6380:6380" - command: - - redis-server - - "--bind 0.0.0.0 ::" - - --requirepass public - - --tls-port 6380 - - --tls-cert-file /etc/certs/redis.crt - - --tls-key-file /etc/certs/redis.key - - --tls-ca-cert-file /etc/certs/ca.crt - - --tls-protocols "TLSv1.3" - - --tls-ciphersuites "TLS_CHACHA20_POLY1305_SHA256" + command: redis-server /usr/local/etc/redis/redis.conf restart: always networks: emqx_bridge: diff --git a/.ci/docker-compose-file/redis/cluster-tcp/redis.conf b/.ci/docker-compose-file/redis/cluster-tcp/redis.conf index 79a0d8a73..6930bde1c 100644 --- a/.ci/docker-compose-file/redis/cluster-tcp/redis.conf +++ b/.ci/docker-compose-file/redis/cluster-tcp/redis.conf @@ -1,10 +1,11 @@ bind :: 0.0.0.0 port 6379 -requirepass public cluster-enabled yes +masteruser default masterauth public +aclfile /usr/local/etc/redis/users.acl protected-mode no daemonize no diff --git a/.ci/docker-compose-file/redis/cluster-tcp/users.acl b/.ci/docker-compose-file/redis/cluster-tcp/users.acl new file mode 100644 index 000000000..5bafe9f6d --- /dev/null +++ b/.ci/docker-compose-file/redis/cluster-tcp/users.acl @@ -0,0 +1,2 @@ +user default on >public ~* &* +@all +user test_user on >test_passwd ~* &* +@all diff --git a/.ci/docker-compose-file/redis/cluster-tls/redis.conf b/.ci/docker-compose-file/redis/cluster-tls/redis.conf index 3020f46a7..5d203de80 100644 --- a/.ci/docker-compose-file/redis/cluster-tls/redis.conf +++ b/.ci/docker-compose-file/redis/cluster-tls/redis.conf @@ -1,10 +1,11 @@ bind :: 0.0.0.0 port 6379 -requirepass public cluster-enabled yes +masteruser default masterauth public +aclfile /usr/local/etc/redis/users.acl tls-port 6389 tls-cert-file /etc/certs/cert.pem diff --git a/.ci/docker-compose-file/redis/cluster-tls/users.acl b/.ci/docker-compose-file/redis/cluster-tls/users.acl new file mode 100644 index 000000000..5bafe9f6d --- /dev/null +++ b/.ci/docker-compose-file/redis/cluster-tls/users.acl @@ -0,0 +1,2 @@ +user default on >public ~* &* +@all +user test_user on >test_passwd ~* &* +@all diff --git a/.ci/docker-compose-file/redis/sentinel-tcp/master.conf b/.ci/docker-compose-file/redis/sentinel-tcp/master.conf index 25940c887..a531d1b40 100644 --- a/.ci/docker-compose-file/redis/sentinel-tcp/master.conf +++ b/.ci/docker-compose-file/redis/sentinel-tcp/master.conf @@ -1,6 +1,6 @@ bind :: 0.0.0.0 port 6379 -requirepass public +aclfile /usr/local/etc/redis/users.acl protected-mode no daemonize no diff --git a/.ci/docker-compose-file/redis/sentinel-tcp/slave.conf b/.ci/docker-compose-file/redis/sentinel-tcp/slave.conf index 2c61aeb6c..4a7e240fc 100644 --- a/.ci/docker-compose-file/redis/sentinel-tcp/slave.conf +++ b/.ci/docker-compose-file/redis/sentinel-tcp/slave.conf @@ -1,9 +1,10 @@ bind :: 0.0.0.0 port 6379 -requirepass public replicaof redis-sentinel-master 6379 +masteruser default masterauth public +aclfile /usr/local/etc/redis/users.acl protected-mode no daemonize no diff --git a/.ci/docker-compose-file/redis/sentinel-tcp/users.acl b/.ci/docker-compose-file/redis/sentinel-tcp/users.acl new file mode 100644 index 000000000..5bafe9f6d --- /dev/null +++ b/.ci/docker-compose-file/redis/sentinel-tcp/users.acl @@ -0,0 +1,2 @@ +user default on >public ~* &* +@all +user test_user on >test_passwd ~* &* +@all diff --git a/.ci/docker-compose-file/redis/sentinel-tls/master.conf b/.ci/docker-compose-file/redis/sentinel-tls/master.conf index f55433f79..68e01f323 100644 --- a/.ci/docker-compose-file/redis/sentinel-tls/master.conf +++ b/.ci/docker-compose-file/redis/sentinel-tls/master.conf @@ -1,6 +1,6 @@ bind :: 0.0.0.0 port 6379 -requirepass public +aclfile /usr/local/etc/redis/users.acl tls-port 6389 tls-cert-file /etc/certs/cert.pem diff --git a/.ci/docker-compose-file/redis/sentinel-tls/slave.conf b/.ci/docker-compose-file/redis/sentinel-tls/slave.conf index d8758da51..25102d5ed 100644 --- a/.ci/docker-compose-file/redis/sentinel-tls/slave.conf +++ b/.ci/docker-compose-file/redis/sentinel-tls/slave.conf @@ -1,9 +1,10 @@ bind :: 0.0.0.0 port 6379 -requirepass public replicaof redis-sentinel-tls-master 6389 +masteruser default masterauth public +aclfile /usr/local/etc/redis/users.acl tls-port 6389 tls-replication yes diff --git a/.ci/docker-compose-file/redis/sentinel-tls/users.acl b/.ci/docker-compose-file/redis/sentinel-tls/users.acl new file mode 100644 index 000000000..5bafe9f6d --- /dev/null +++ b/.ci/docker-compose-file/redis/sentinel-tls/users.acl @@ -0,0 +1,2 @@ +user default on >public ~* &* +@all +user test_user on >test_passwd ~* &* +@all diff --git a/.ci/docker-compose-file/redis/single-tcp/redis.conf b/.ci/docker-compose-file/redis/single-tcp/redis.conf new file mode 100644 index 000000000..1e5f629d2 --- /dev/null +++ b/.ci/docker-compose-file/redis/single-tcp/redis.conf @@ -0,0 +1,3 @@ +bind :: 0.0.0.0 +port 6379 +aclfile /usr/local/etc/redis/users.acl diff --git a/.ci/docker-compose-file/redis/single-tcp/users.acl b/.ci/docker-compose-file/redis/single-tcp/users.acl new file mode 100644 index 000000000..5bafe9f6d --- /dev/null +++ b/.ci/docker-compose-file/redis/single-tcp/users.acl @@ -0,0 +1,2 @@ +user default on >public ~* &* +@all +user test_user on >test_passwd ~* &* +@all diff --git a/.ci/docker-compose-file/redis/single-tls/redis.conf b/.ci/docker-compose-file/redis/single-tls/redis.conf new file mode 100644 index 000000000..e2d40bca6 --- /dev/null +++ b/.ci/docker-compose-file/redis/single-tls/redis.conf @@ -0,0 +1,9 @@ +bind :: 0.0.0.0 +aclfile /usr/local/etc/redis/users.acl + +tls-port 6380 +tls-cert-file /etc/certs/redis.crt +tls-key-file /etc/certs/redis.key +tls-ca-cert-file /etc/certs/ca.crt +tls-protocols "TLSv1.3" +tls-ciphersuites "TLS_CHACHA20_POLY1305_SHA256" diff --git a/.ci/docker-compose-file/redis/single-tls/users.acl b/.ci/docker-compose-file/redis/single-tls/users.acl new file mode 100644 index 000000000..5bafe9f6d --- /dev/null +++ b/.ci/docker-compose-file/redis/single-tls/users.acl @@ -0,0 +1,2 @@ +user default on >public ~* &* +@all +user test_user on >test_passwd ~* &* +@all diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src index b380bc86d..5b6163969 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_redis, [ {description, "EMQX Enterprise Redis Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl b/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl index 6a0248b67..0c17ab775 100644 --- a/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl +++ b/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl @@ -30,6 +30,11 @@ <<"local_topic">> => <<"local_topic/#">> }). +-define(USERNAME_PASSWORD_AUTH_OPTS, #{ + <<"username">> => <<"test_user">>, + <<"password">> => <<"test_passwd">> +}). + -define(BATCH_SIZE, 5). -define(PROXY_HOST, "toxiproxy"). @@ -319,6 +324,22 @@ t_permanent_error(_Config) -> ), {ok, _} = emqx_bridge:remove(Type, Name). +t_auth_username_password(_Config) -> + Name = <<"mybridge">>, + Type = <<"redis_single">>, + ResourceId = emqx_bridge_resource:resource_id(Type, Name), + BridgeConfig = username_password_redis_bridge_config(), + ?assertMatch( + {ok, _}, + emqx_bridge:create(Type, Name, BridgeConfig) + ), + ?WAIT( + {ok, connected}, + emqx_resource:health_check(ResourceId), + 5 + ), + {ok, _} = emqx_bridge:remove(Type, Name). + t_create_disconnected(Config) -> Name = <<"toxic_bridge">>, Type = <<"redis_single">>, @@ -528,6 +549,19 @@ toxiproxy_redis_bridge_config() -> }, maps:merge(Conf0, ?COMMON_REDIS_OPTS). +username_password_redis_bridge_config() -> + Conf0 = ?REDIS_TOXYPROXY_CONNECT_CONFIG#{ + <<"resource_opts">> => #{ + <<"query_mode">> => <<"sync">>, + <<"worker_pool_size">> => <<"1">>, + <<"batch_size">> => integer_to_binary(?BATCH_SIZE), + <<"health_check_interval">> => <<"1s">>, + <<"start_timeout">> => <<"15s">> + } + }, + Conf1 = maps:merge(Conf0, ?COMMON_REDIS_OPTS), + maps:merge(Conf1, ?USERNAME_PASSWORD_AUTH_OPTS). + invalid_command_bridge_config() -> #{redis_single := #{tcp := Conf0}} = redis_connect_configs(), Conf1 = maps:merge(Conf0, ?COMMON_REDIS_OPTS), diff --git a/apps/emqx_redis/rebar.config b/apps/emqx_redis/rebar.config index c14536384..4e67e0986 100644 --- a/apps/emqx_redis/rebar.config +++ b/apps/emqx_redis/rebar.config @@ -3,7 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ %% NOTE: mind ecpool version when updating eredis_cluster version - {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.8.1"}}}, + {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.8.2"}}}, {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}} ]}. diff --git a/apps/emqx_redis/src/emqx_redis.app.src b/apps/emqx_redis/src/emqx_redis.app.src index 294a642f5..23e13bb72 100644 --- a/apps/emqx_redis/src/emqx_redis.app.src +++ b/apps/emqx_redis/src/emqx_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_redis, [ {description, "EMQX Redis Database Connector"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_redis/src/emqx_redis.erl b/apps/emqx_redis/src/emqx_redis.erl index ef89c3931..c76cbdfb4 100644 --- a/apps/emqx_redis/src/emqx_redis.erl +++ b/apps/emqx_redis/src/emqx_redis.erl @@ -146,6 +146,7 @@ on_start( Opts = [ {pool_size, PoolSize}, + {username, maps:get(username, Config, undefined)}, {password, maps:get(password, Config, "")}, {auto_reconnect, ?AUTO_RECONNECT_INTERVAL} ] ++ Database ++ Servers, @@ -292,6 +293,7 @@ connect(Opts) -> redis_fields() -> [ {pool_size, fun emqx_connector_schema_lib:pool_size/1}, + {username, fun emqx_connector_schema_lib:username/1}, {password, fun emqx_connector_schema_lib:password/1}, {database, #{ type => non_neg_integer(), diff --git a/apps/emqx_redis/test/emqx_redis_SUITE.erl b/apps/emqx_redis/test/emqx_redis_SUITE.erl index c425f19d9..e03b05921 100644 --- a/apps/emqx_redis/test/emqx_redis_SUITE.erl +++ b/apps/emqx_redis/test/emqx_redis_SUITE.erl @@ -137,6 +137,31 @@ perform_lifecycle_check(ResourceId, InitialConfig, RedisCommand) -> #{timeout => 500} ) ), + % check authentication methods + ?assertEqual( + {ok, <<"OK">>}, + emqx_resource:query(ResourceId, {cmd, ["AUTH", "public"]}) + ), + ?assertEqual( + {error, <<"WRONGPASS invalid username-password pair or user is disabled.">>}, + emqx_resource:query(ResourceId, {cmd, ["AUTH", "test_passwd"]}) + ), + ?assertEqual( + {ok, <<"OK">>}, + emqx_resource:query(ResourceId, {cmd, ["AUTH", "test_user", "test_passwd"]}) + ), + ?assertEqual( + {error, <<"WRONGPASS invalid username-password pair or user is disabled.">>}, + emqx_resource:query(ResourceId, {cmd, ["AUTH", "test_user", "public"]}) + ), + ?assertEqual( + {error, <<"WRONGPASS invalid username-password pair or user is disabled.">>}, + emqx_resource:query(ResourceId, {cmd, ["AUTH", "wrong_user", "test_passwd"]}) + ), + ?assertEqual( + {error, <<"WRONGPASS invalid username-password pair or user is disabled.">>}, + emqx_resource:query(ResourceId, {cmd, ["AUTH", "wrong_user", "public"]}) + ), ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. @@ -186,7 +211,8 @@ redis_config_sentinel() -> " redis_type = ~s\n" ++ MaybeSentinel ++ MaybeDatabase ++ - " password = public\n" ++ + " username = test_user\n" ++ + " password = test_passwd\n" ++ " ~s = \"~s:~b\"\n" ++ " " ++ "" diff --git a/changes/ce/feat-11469.en.md b/changes/ce/feat-11469.en.md new file mode 100644 index 000000000..827fe3a87 --- /dev/null +++ b/changes/ce/feat-11469.en.md @@ -0,0 +1 @@ +Added support for specifying username in Redis authentication. From fc35c74eab3d8465bc246ac5397cfdba36ff3f01 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Mon, 14 Aug 2023 21:22:31 -0300 Subject: [PATCH 022/119] fix(redis): use eredis_secret to avoid password leaking --- apps/emqx_redis/src/emqx_redis.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_redis/src/emqx_redis.erl b/apps/emqx_redis/src/emqx_redis.erl index c76cbdfb4..2779620bf 100644 --- a/apps/emqx_redis/src/emqx_redis.erl +++ b/apps/emqx_redis/src/emqx_redis.erl @@ -147,7 +147,7 @@ on_start( [ {pool_size, PoolSize}, {username, maps:get(username, Config, undefined)}, - {password, maps:get(password, Config, "")}, + {password, eredis_secret:wrap(maps:get(password, Config, ""))}, {auto_reconnect, ?AUTO_RECONNECT_INTERVAL} ] ++ Database ++ Servers, Options = From 05fdf52d8eef1be357992879a983ee55b23f4f92 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Fri, 18 Aug 2023 13:26:31 -0300 Subject: [PATCH 023/119] fix(redis): set bridge as unhealthy on authentication error --- .../src/emqx_bridge_redis_connector.erl | 6 +++ .../test/emqx_bridge_redis_SUITE.erl | 41 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl index 38a80048e..696947726 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl @@ -35,6 +35,12 @@ on_start(InstId, #{command_template := CommandTemplate} = Config) -> conn_st => RedisConnSt, command_template => preproc_command_template(CommandTemplate) }}; + {error, {start_pool_failed, _, #{type := authentication_error, reason := Reason}}} = Error -> + ?tp( + redis_bridge_connector_start_error, + #{error => Error} + ), + throw({unhealthy_target, Reason}); {error, _} = Error -> ?tp( redis_bridge_connector_start_error, diff --git a/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl b/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl index 0c17ab775..c4089323b 100644 --- a/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl +++ b/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl @@ -340,6 +340,47 @@ t_auth_username_password(_Config) -> ), {ok, _} = emqx_bridge:remove(Type, Name). +t_auth_error_username_password(_Config) -> + Name = <<"mybridge">>, + Type = <<"redis_single">>, + ResourceId = emqx_bridge_resource:resource_id(Type, Name), + BridgeConfig0 = username_password_redis_bridge_config(), + BridgeConfig = maps:merge(BridgeConfig0, #{<<"password">> => <<"wrong_password">>}), + ?assertMatch( + {ok, _}, + emqx_bridge:create(Type, Name, BridgeConfig) + ), + ?WAIT( + {ok, disconnected}, + emqx_resource:health_check(ResourceId), + 5 + ), + ?assertMatch( + {ok, _, #{error := {unhealthy_target, _Msg}}}, + emqx_resource_manager:lookup(ResourceId) + ), + {ok, _} = emqx_bridge:remove(Type, Name). + +t_auth_error_password_only(_Config) -> + Name = <<"mybridge">>, + Type = <<"redis_single">>, + ResourceId = emqx_bridge_resource:resource_id(Type, Name), + BridgeConfig0 = toxiproxy_redis_bridge_config(), + BridgeConfig = maps:merge(BridgeConfig0, #{<<"password">> => <<"wrong_password">>}), + ?assertMatch( + {ok, _}, + emqx_bridge:create(Type, Name, BridgeConfig) + ), + ?assertEqual( + {ok, disconnected}, + emqx_resource:health_check(ResourceId) + ), + ?assertMatch( + {ok, _, #{error := {unhealthy_target, _Msg}}}, + emqx_resource_manager:lookup(ResourceId) + ), + {ok, _} = emqx_bridge:remove(Type, Name). + t_create_disconnected(Config) -> Name = <<"toxic_bridge">>, Type = <<"redis_single">>, From 5527edf4426f9ac10d9e556de1444b025423785c Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 15 Aug 2023 16:56:25 +0800 Subject: [PATCH 024/119] chore: bump hstreamdb_erl to `0.4.3+v0.16.1` --- apps/emqx_bridge_hstreamdb/rebar.config | 2 +- .../src/emqx_bridge_hstreamdb.app.src | 2 +- .../src/emqx_bridge_hstreamdb_connector.erl | 108 +++++++++--------- .../test/emqx_bridge_hstreamdb_SUITE.erl | 6 +- mix.exs | 2 +- 5 files changed, 63 insertions(+), 57 deletions(-) diff --git a/apps/emqx_bridge_hstreamdb/rebar.config b/apps/emqx_bridge_hstreamdb/rebar.config index 9a70b55f9..44970b75d 100644 --- a/apps/emqx_bridge_hstreamdb/rebar.config +++ b/apps/emqx_bridge_hstreamdb/rebar.config @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {erl_opts, [debug_info]}. {deps, [ - {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.3.1+v0.12.0"}}}, + {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.4.3+v0.16.1"}}}, {emqx, {path, "../../apps/emqx"}}, {emqx_utils, {path, "../../apps/emqx_utils"}} ]}. diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src index 2a800baca..39c8c3258 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_hstreamdb, [ {description, "EMQX Enterprise HStreamDB Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl index 1763b252a..727ea4ad8 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl @@ -75,7 +75,7 @@ on_query( } ) -> try to_record(PartitionKey, HRecordTemplate, Data) of - Record -> append_record(Producer, Record) + Record -> append_record(Producer, Record, false) catch _:_ -> ?FAILED_TO_APPLY_HRECORD_TEMPLATE end. @@ -88,7 +88,7 @@ on_batch_query( } ) -> try to_multi_part_records(PartitionKey, HRecordTemplate, BatchList) of - Records -> append_record(Producer, Records) + Records -> append_record(Producer, Records, true) catch _:_ -> ?FAILED_TO_APPLY_HRECORD_TEMPLATE end. @@ -156,16 +156,29 @@ start_client(InstId, Config) -> {error, Error} end. -do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) -> +do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize, ssl := SSL}) -> ?SLOG(info, #{ msg => "starting hstreamdb connector: client", connector => InstId, config => Config }), ClientName = client_name(InstId), + RpcOpts = + case maps:get(enable, SSL) of + false -> + #{pool_size => PoolSize}; + true -> + #{ + pool_size => PoolSize, + gun_opts => #{ + transport => tls, + transport_opts => emqx_tls_lib:to_client_opts(SSL) + } + } + end, ClientOptions = [ {url, binary_to_list(Server)}, - {rpc_options, #{pool_size => PoolSize}} + {rpc_options, RpcOpts} ], case hstreamdb:start_client(ClientName, ClientOptions) of {ok, Client} -> @@ -206,12 +219,7 @@ do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) -> end. is_alive(Client) -> - case hstreamdb:echo(Client) of - {ok, _Echo} -> - true; - _ErrorEcho -> - false - end. + hstreamdb_client:echo(Client) =:= ok. start_producer( InstId, @@ -280,54 +288,52 @@ to_record(PartitionKey, RawRecord) -> hstreamdb:to_record(PartitionKey, raw, RawRecord). to_multi_part_records(PartitionKeyTmpl, HRecordTmpl, BatchList) -> - Records0 = lists:map( + lists:map( fun({send_message, Data}) -> to_record(PartitionKeyTmpl, HRecordTmpl, Data) end, BatchList - ), - PartitionKeys = proplists:get_keys(Records0), - [ - {PartitionKey, proplists:get_all_values(PartitionKey, Records0)} - || PartitionKey <- PartitionKeys - ]. + ). -append_record(Producer, MultiPartsRecords) when is_list(MultiPartsRecords) -> - lists:foreach(fun(Record) -> append_record(Producer, Record) end, MultiPartsRecords); -append_record(Producer, Record) when is_tuple(Record) -> - do_append_records(false, Producer, Record). +append_record(Producer, MultiPartsRecords, MaybeBatch) when is_list(MultiPartsRecords) -> + lists:foreach( + fun(Record) -> append_record(Producer, Record, MaybeBatch) end, MultiPartsRecords + ); +append_record(Producer, Record, MaybeBatch) when is_tuple(Record) -> + do_append_records(Producer, Record, MaybeBatch). %% TODO: only sync request supported. implement async request later. -do_append_records(false, Producer, Record) -> - case hstreamdb:append_flush(Producer, Record) of - {ok, _Result} -> - ?tp( - hstreamdb_connector_query_return, - #{result => _Result} - ), - ?SLOG(debug, #{ - msg => "HStreamDB producer sync append success", - record => Record - }); - %% the HStream is warming up or buzy, something are not ready yet, retry after a while - {error, {unavailable, _} = Reason} -> - {error, - {recoverable_error, #{ - msg => "HStreamDB is warming up or buzy, will retry after a moment", - reason => Reason - }}}; - {error, Reason} = Err -> - ?tp( - hstreamdb_connector_query_return, - #{error => Reason} - ), - ?SLOG(error, #{ - msg => "HStreamDB producer sync append failed", - reason => Reason, - record => Record - }), - Err - end. +do_append_records(Producer, Record, true = IsBatch) -> + Result = hstreamdb:append(Producer, Record), + handle_result(Result, Record, IsBatch); +do_append_records(Producer, Record, false = IsBatch) -> + Result = hstreamdb:append_flush(Producer, Record), + handle_result(Result, Record, IsBatch). + +handle_result(ok = Result, Record, IsBatch) -> + handle_result({ok, Result}, Record, IsBatch); +handle_result({ok, Result}, Record, IsBatch) -> + ?tp( + hstreamdb_connector_query_append_return, + #{result => Result, is_batch => IsBatch} + ), + ?SLOG(debug, #{ + msg => "HStreamDB producer sync append success", + record => Record, + is_batch => IsBatch + }); +handle_result({error, Reason} = Err, Record, IsBatch) -> + ?tp( + hstreamdb_connector_query_append_return, + #{error => Reason, is_batch => IsBatch} + ), + ?SLOG(error, #{ + msg => "HStreamDB producer sync append failed", + reason => Reason, + record => Record, + is_batch => IsBatch + }), + Err. client_name(InstId) -> "client:" ++ to_string(InstId). diff --git a/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl index 430343274..46f28ad40 100644 --- a/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl +++ b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl @@ -432,7 +432,7 @@ client(Name, Config, N) -> try _ = hstreamdb:stop_client(Name), {ok, Client} = hstreamdb:start_client(Name, default_options(Config)), - {ok, echo} = hstreamdb:echo(Client), + ok = hstreamdb_client:echo(Client), Client catch Class:Error -> @@ -509,7 +509,7 @@ health_check_resource_down(Config) -> % These funs start and then stop the hstreamdb connection connect_and_create_stream(Config) -> ?WITH_CLIENT( - _ = hstreamdb:create_stream( + _ = hstreamdb_client:create_stream( Client, ?STREAM, ?REPLICATION_FACTOR, ?BACKLOG_RETENTION_SECOND, ?SHARD_COUNT ) ), @@ -531,7 +531,7 @@ connect_and_create_stream(Config) -> connect_and_delete_stream(Config) -> ?WITH_CLIENT( - _ = hstreamdb:delete_stream(Client, ?STREAM) + _ = hstreamdb_client:delete_stream(Client, ?STREAM) ). %%-------------------------------------------------------------------- diff --git a/mix.exs b/mix.exs index e6047249e..496f0b0c5 100644 --- a/mix.exs +++ b/mix.exs @@ -227,7 +227,7 @@ defmodule EMQXUmbrella.MixProject do defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do [ - {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.3.1+v0.12.0"}, + {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.4.3+v0.16.1"}, {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.11", override: true}, {:wolff, github: "kafka4beam/wolff", tag: "1.7.6"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.3", override: true}, From 649647190e686ef17c0f2b805f1697fed9e88225 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 18 Aug 2023 18:10:29 +0800 Subject: [PATCH 025/119] test: new batch mechanism of hstreamdb_erl --- .../test/emqx_bridge_hstreamdb_SUITE.erl | 71 ++++++++++++++----- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl index 46f28ad40..14ea202be 100644 --- a/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl +++ b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl @@ -13,8 +13,13 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). % SQL definitions --define(STREAM, "stream"). + +-define(STREAM, "demo_stream"). +%% could not be "stream" in Production Environment +%% especially not in hstreamdb_sql CLI client + -define(REPLICATION_FACTOR, 1). + %% in seconds -define(BACKLOG_RETENTION_SECOND, (24 * 60 * 60)). -define(SHARD_COUNT, 1). @@ -146,16 +151,23 @@ t_setup_via_config_and_publish(Config) -> begin ?wait_async_action( ?assertEqual(ok, send_message(Config, Data)), - #{?snk_kind := hstreamdb_connector_query_return}, + #{?snk_kind := hstreamdb_connector_query_append_return}, 10_000 ), ok end, fun(Trace0) -> - Trace = ?of_kind(hstreamdb_connector_query_return, Trace0), + Trace = ?of_kind(hstreamdb_connector_query_append_return, Trace0), lists:foreach( fun(EachTrace) -> - ?assertMatch(#{result := #{streamName := <>}}, EachTrace) + case ?config(enable_batch, Config) of + true -> + ?assertMatch(#{result := ok, is_batch := true}, EachTrace); + false -> + ?assertMatch( + #{result := #{'batchId' := _}, is_batch := false}, EachTrace + ) + end end, Trace ), @@ -181,16 +193,26 @@ t_setup_via_http_api_and_publish(Config) -> begin ?wait_async_action( ?assertEqual(ok, send_message(Config, Data)), - #{?snk_kind := hstreamdb_connector_query_return}, + #{?snk_kind := hstreamdb_connector_query_append_return}, 10_000 ), ok end, fun(Trace) -> - ?assertMatch( - [#{result := #{streamName := <>}}], - ?of_kind(hstreamdb_connector_query_return, Trace) - ) + lists:foreach( + fun(EachTrace) -> + case ?config(enable_batch, Config) of + true -> + ?assertMatch(#{result := ok, is_batch := true}, EachTrace); + false -> + ?assertMatch( + #{result := #{'batchId' := _}, is_batch := false}, EachTrace + ) + end + end, + ?of_kind(hstreamdb_connector_query_append_return, Trace) + ), + ok end ), ok. @@ -240,6 +262,7 @@ t_write_failure(Config) -> ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), QueryMode = ?config(query_mode, Config), + EnableBatch = ?config(enable_batch, Config), Data = rand_data(), {{ok, _}, {ok, _}} = ?wait_async_action( @@ -251,10 +274,16 @@ t_write_failure(Config) -> health_check_resource_down(Config), case QueryMode of sync -> - ?assertMatch( - {error, {resource_error, #{msg := "call resource timeout", reason := timeout}}}, - send_message(Config, Data) - ); + case EnableBatch of + true -> + %% append to batch always returns ok + ?assertMatch(ok, send_message(Config, Data)); + false -> + ?assertMatch( + {error, {cannot_list_shards, {<>, econnrefused}}}, + send_message(Config, Data) + ) + end; async -> %% TODO: async mode is not supported yet, %% but it will return ok if calling emqx_resource_buffer_worker:async_query/3, @@ -282,17 +311,23 @@ t_simple_query(Config) -> end, Requests ), - #{?snk_kind := hstreamdb_connector_query_return}, + #{?snk_kind := hstreamdb_connector_query_append_return}, 10_000 ) end, - fun(Trace0) -> - Trace = ?of_kind(hstreamdb_connector_query_return, Trace0), + fun(Trace) -> lists:foreach( fun(EachTrace) -> - ?assertMatch(#{result := #{streamName := <>}}, EachTrace) + case ?config(enable_batch, Config) of + true -> + ?assertMatch(#{result := ok, is_batch := true}, EachTrace); + false -> + ?assertMatch( + #{result := #{'batchId' := _}, is_batch := false}, EachTrace + ) + end end, - Trace + ?of_kind(hstreamdb_connector_query_append_return, Trace) ), ok end From f45b5fac6f3cb844b093b219a53447167660e17c Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 18 Aug 2023 18:20:37 +0800 Subject: [PATCH 026/119] chore: unhidden hstreamdb bridge for e5.2.0 --- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl | 6 ++---- changes/ee/feat-11478.en.md | 1 + 3 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 changes/ee/feat-11478.en.md diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 96d953e34..3a76843a1 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.26"}, + {vsn, "0.1.27"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl index 07f858979..a6bd4a754 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl @@ -32,8 +32,7 @@ api_schemas(Method) -> api_ref(emqx_bridge_mongodb, <<"mongodb_rs">>, Method ++ "_rs"), api_ref(emqx_bridge_mongodb, <<"mongodb_sharded">>, Method ++ "_sharded"), api_ref(emqx_bridge_mongodb, <<"mongodb_single">>, Method ++ "_single"), - %% TODO: un-hide for e5.2.0... - %%api_ref(emqx_bridge_hstreamdb, <<"hstreamdb">>, Method), + api_ref(emqx_bridge_hstreamdb, <<"hstreamdb">>, Method), api_ref(emqx_bridge_influxdb, <<"influxdb_api_v1">>, Method ++ "_api_v1"), api_ref(emqx_bridge_influxdb, <<"influxdb_api_v2">>, Method ++ "_api_v2"), api_ref(emqx_bridge_redis, <<"redis_single">>, Method ++ "_single"), @@ -147,8 +146,7 @@ fields(bridges) -> hoconsc:map(name, ref(emqx_bridge_hstreamdb, "config")), #{ desc => <<"HStreamDB Bridge Config">>, - required => false, - importance => ?IMPORTANCE_HIDDEN + required => false } )}, {mysql, diff --git a/changes/ee/feat-11478.en.md b/changes/ee/feat-11478.en.md new file mode 100644 index 000000000..42761c77e --- /dev/null +++ b/changes/ee/feat-11478.en.md @@ -0,0 +1 @@ +Add HStreamDB bridge support (both TCP and TLS connection allowed), adapted to the HStreamDB `v0.16.1`. From 443d44aaa3b09fc29273f173c0c1444406a37ed9 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Mon, 21 Aug 2023 11:49:57 +0800 Subject: [PATCH 027/119] fix: set the `endpoint` in kinesis bridge as a required field --- apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl index 5ed60c8aa..92faa2279 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl @@ -55,7 +55,8 @@ fields(connector_config) -> mk( binary(), #{ - default => <<"https://kinesis.us-east-1.amazonaws.com">>, + required => true, + example => <<"https://kinesis.us-east-1.amazonaws.com">>, desc => ?DESC("endpoint") } )}, From f559ac3410818af7eb059c609dfffcf85baeb3a6 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 21 Aug 2023 17:17:19 +0800 Subject: [PATCH 028/119] ci: bump hstreamdb & zookeeper image vsn --- .ci/docker-compose-file/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index b7033caae..e65600f06 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -10,7 +10,7 @@ CASSANDRA_TAG=3.11.6 MINIO_TAG=RELEASE.2023-03-20T20-16-18Z OPENTS_TAG=9aa7f88 KINESIS_TAG=2.1 -HSTREAMDB_TAG=v0.15.0 +HSTREAMDB_TAG=v0.16.1 HSTREAMDB_ZK_TAG=3.8.1 MS_IMAGE_ADDR=mcr.microsoft.com/mssql/server From 037b0a7599a6f511be931198a907c5ab9e676fd0 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 21 Aug 2023 22:47:18 +0800 Subject: [PATCH 029/119] fix(ldap): Return correct reason code when password is wrong --- apps/emqx_ldap/src/emqx_ldap_authn.erl | 2 +- apps/emqx_ldap/test/emqx_ldap_authn_SUITE.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ldap/src/emqx_ldap_authn.erl b/apps/emqx_ldap/src/emqx_ldap_authn.erl index d814e2aae..8d7997567 100644 --- a/apps/emqx_ldap/src/emqx_ldap_authn.erl +++ b/apps/emqx_ldap/src/emqx_ldap_authn.erl @@ -249,7 +249,7 @@ verify_password(Algorithm, LDAPPasswordType, LDAPPassword, Salt, Position, Passw true -> {ok, is_superuser(Entry, State)}; _ -> - {error, invalid_password} + {error, bad_username_or_password} end. is_superuser(Entry, #{is_superuser_attribute := Attr} = _State) -> diff --git a/apps/emqx_ldap/test/emqx_ldap_authn_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_authn_SUITE.erl index 5167d2267..40501456e 100644 --- a/apps/emqx_ldap/test/emqx_ldap_authn_SUITE.erl +++ b/apps/emqx_ldap/test/emqx_ldap_authn_SUITE.erl @@ -237,7 +237,7 @@ user_seeds() -> %% Not exists New(<<"notexists">>, <<"notexists">>, {error, not_authorized}), %% Wrong Password - New(<<"mqttuser0001">>, <<"wrongpassword">>, {error, invalid_password}), + New(<<"mqttuser0001">>, <<"wrongpassword">>, {error, bad_username_or_password}), %% Disabled New(<<"mqttuser0006">>, <<"mqttuser0006">>, {error, user_disabled}), %% IsSuperuser From b0c5b447a442b2016d4e69690efb1898695f7d8e Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 22 Aug 2023 11:49:32 +0800 Subject: [PATCH 030/119] fix(authn): quickly return when the password is absent in password-based authentication --- apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl | 2 ++ apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl | 2 ++ apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl | 2 ++ apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl | 2 ++ apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl | 2 ++ apps/emqx_ldap/src/emqx_ldap_authn.erl | 2 ++ 6 files changed, 12 insertions(+) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index bf0b04d04..2a124ae98 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -173,6 +173,8 @@ update(Config, _State) -> authenticate(#{auth_method := _}, _) -> ignore; +authenticate(#{password := undefined}, _) -> + {error, bad_username_or_password}; authenticate( #{password := Password} = Credential, #{ diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 4498d3d8d..9cbd1f2dc 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -160,6 +160,8 @@ destroy(#{resource_id := ResourceId}) -> authenticate(#{auth_method := _}, _) -> ignore; +authenticate(#{password := undefined}, _) -> + {error, bad_username_or_password}; authenticate( #{password := Password} = Credential, #{ diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index dc4e0d163..49471eb23 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -110,6 +110,8 @@ destroy(#{resource_id := ResourceId}) -> authenticate(#{auth_method := _}, _) -> ignore; +authenticate(#{password := undefined}, _) -> + {error, bad_username_or_password}; authenticate( #{password := Password} = Credential, #{ diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index d9526cc7b..b9ce9db8d 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -113,6 +113,8 @@ destroy(#{resource_id := ResourceId}) -> authenticate(#{auth_method := _}, _) -> ignore; +authenticate(#{password := undefined}, _) -> + {error, bad_username_or_password}; authenticate( #{password := Password} = Credential, #{ diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index f6f02c1bc..a5312e41b 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -148,6 +148,8 @@ destroy(#{resource_id := ResourceId}) -> authenticate(#{auth_method := _}, _) -> ignore; +authenticate(#{password := undefined}, _) -> + {error, bad_username_or_password}; authenticate( #{password := Password} = Credential, #{ diff --git a/apps/emqx_ldap/src/emqx_ldap_authn.erl b/apps/emqx_ldap/src/emqx_ldap_authn.erl index d814e2aae..b600e10b6 100644 --- a/apps/emqx_ldap/src/emqx_ldap_authn.erl +++ b/apps/emqx_ldap/src/emqx_ldap_authn.erl @@ -109,6 +109,8 @@ destroy(#{resource_id := ResourceId}) -> authenticate(#{auth_method := _}, _) -> ignore; +authenticate(#{password := undefined}, _) -> + {error, bad_username_or_password}; authenticate( #{password := Password} = Credential, #{ From 184e03387aae9b65280482f9ede45a6da9d3123d Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 22 Aug 2023 15:44:40 +0800 Subject: [PATCH 031/119] fix(ldap): fix status detection and auto reconnecting errors --- .../docker-compose-ldap.yaml | 2 + .ci/docker-compose-file/toxiproxy.json | 12 ++++++ apps/emqx_ldap/src/emqx_ldap.erl | 11 ++++- apps/emqx_ldap/test/emqx_ldap_SUITE.erl | 41 +++++++++++++++++-- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose-ldap.yaml b/.ci/docker-compose-file/docker-compose-ldap.yaml index e6c8ba2d8..f92df47a0 100644 --- a/.ci/docker-compose-file/docker-compose-ldap.yaml +++ b/.ci/docker-compose-file/docker-compose-ldap.yaml @@ -11,6 +11,8 @@ services: image: openldap #ports: # - 389:389 + volumes: + - ./certs/ca.crt:/etc/certs/ca.crt restart: always networks: - emqx_bridge diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index f5df5a853..4b2b6ccf2 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -179,5 +179,17 @@ "listen": "0.0.0.0:4566", "upstream": "kinesis:4566", "enabled": true + }, + { + "name": "ldap_tcp", + "listen": "0.0.0.0:389", + "upstream": "ldap:389", + "enabled": true + }, + { + "name": "ldap_ssl", + "listen": "0.0.0.0:636", + "upstream": "ldap:636", + "enabled": true } ] diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index 85ba73df9..73a0a363c 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -145,11 +145,18 @@ on_get_status(_InstId, #{pool_name := PoolName} = _State) -> true -> connected; false -> - connecting + %% Note: here can only return `disconnected` not `connecting` + %% because the LDAP socket/connection can't be reused + %% searching on a died socket will never return until timeout + disconnected end. do_get_status(Conn) -> - erlang:is_process_alive(Conn). + %% search with an invalid base object + %% if the server is down, the result is {error, ldap_closed} + %% otherwise is {error, invalidDNSyntax/timeout} + {error, ldap_closed} =/= + eldap:search(Conn, [{base, "checkalive"}, {filter, eldap:'approxMatch'("", "")}]). %% =================================================================== diff --git a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl index 8a90f6f02..79c549c22 100644 --- a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl +++ b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl @@ -9,12 +9,13 @@ -include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/assert.hrl"). -include_lib("eldap/include/eldap.hrl"). --define(LDAP_HOST, "ldap"). -define(LDAP_RESOURCE_MOD, emqx_ldap). +-define(PROXY_HOST, "toxiproxy"). +-define(PROXY_PORT, 8474). +-define(LDAP_HOST, ?PROXY_HOST). all() -> [ @@ -53,9 +54,11 @@ end_per_suite(_Config) -> _ = application:stop(emqx_connector). init_per_testcase(_, Config) -> + emqx_common_test_helpers:reset_proxy(?PROXY_HOST, ?PROXY_PORT), Config. end_per_testcase(_, _Config) -> + emqx_common_test_helpers:reset_proxy(?PROXY_HOST, ?PROXY_PORT), ok. % %%------------------------------------------------------------------------------ @@ -142,6 +145,31 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> % Should not even be able to get the resource data out of ets now unlike just stopping. ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceId)). +t_get_status(Config) -> + ResourceId = <<"emqx_ldap_status">>, + ProxyName = proxy_name(Config), + + {ok, #{config := CheckedConfig}} = emqx_resource:check_config( + ?LDAP_RESOURCE_MOD, ldap_config(Config) + ), + {ok, _} = emqx_resource:create_local( + ResourceId, + ?CONNECTOR_RESOURCE_GROUP, + ?LDAP_RESOURCE_MOD, + CheckedConfig, + #{} + ), + + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), + emqx_common_test_helpers:with_failure(down, ProxyName, ?PROXY_HOST, ?PROXY_PORT, fun() -> + ?assertMatch( + {ok, Status} when Status =:= disconnected, + emqx_resource:health_check(ResourceId) + ) + end), + ?assertEqual(ok, emqx_resource:remove_local(ResourceId)), + ok. + % %%------------------------------------------------------------------------------ % %% Helpers % %%------------------------------------------------------------------------------ @@ -190,5 +218,12 @@ ssl(Config) -> "ssl.enable=false"; ssl -> "ssl.enable=true\n" - "ssl.cacertfile=\"etc/openldap/cacert.pem\"" + "ssl.cacertfile=\"/etc/certs/ca.crt\"" end. + +proxy_name(tcp) -> + "ldap_tcp"; +proxy_name(ssl) -> + "ldap_ssl"; +proxy_name(Config) -> + proxy_name(proplists:get_value(group, Config, tcp)). From e20b804e6ba88d6f58f91417b2f2c718c2be04df Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 22 Aug 2023 12:02:31 +0800 Subject: [PATCH 032/119] chore: update changes && test cases --- apps/emqx_authn/test/emqx_authn_SUITE.erl | 4 ++-- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 4 ++-- apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl | 2 +- changes/ce/perf-11490.en.md | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 changes/ce/perf-11490.en.md diff --git a/apps/emqx_authn/test/emqx_authn_SUITE.erl b/apps/emqx_authn/test/emqx_authn_SUITE.erl index d5df4add3..b3c786875 100644 --- a/apps/emqx_authn/test/emqx_authn_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_SUITE.erl @@ -102,7 +102,7 @@ t_will_message_connection_denied(Config) when is_list(Config) -> {error, _} = emqtt:connect(Publisher), receive {'DOWN', Ref, process, Publisher, Reason} -> - ?assertEqual({shutdown, unauthorized_client}, Reason) + ?assertEqual({shutdown, malformed_username_or_password}, Reason) after 2000 -> error(timeout) end, @@ -151,7 +151,7 @@ t_password_undefined(Config) when is_list(Config) -> header = #mqtt_packet_header{type = ?CONNACK}, variable = #mqtt_packet_connack{ ack_flags = 0, - reason_code = ?CONNACK_AUTH + reason_code = ?CONNACK_CREDENTIALS }, payload = undefined }, diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 5e740de15..4056f7f84 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -359,7 +359,7 @@ test_authenticator_users(PathPrefix) -> <<"metrics">> := #{ <<"total">> := 1, <<"success">> := 0, - <<"nomatch">> := 1 + <<"failed">> := 1 } } = emqx_utils_json:decode(PageData0, [return_maps]); ["listeners", 'tcp:default'] -> @@ -417,7 +417,7 @@ test_authenticator_users(PathPrefix) -> <<"metrics">> := #{ <<"total">> := 2, <<"success">> := 1, - <<"nomatch">> := 1 + <<"failed">> := 1 } } = emqx_utils_json:decode(PageData01, [return_maps]); ["listeners", 'tcp:default'] -> diff --git a/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl b/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl index ae2cc436e..63cdb3f5f 100644 --- a/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl @@ -102,7 +102,7 @@ t_enable_authn(_Config) -> %% enable_authn set to true, we go to the set up authn and fail {ok, ConnPid1} = emqtt:start_link([{port, 18830}, {clientid, <<"clientid">>}]), ?assertMatch( - {error, {unauthorized_client, _}}, + {error, {malformed_username_or_password, _}}, emqtt:connect(ConnPid1) ), ok. diff --git a/changes/ce/perf-11490.en.md b/changes/ce/perf-11490.en.md new file mode 100644 index 000000000..98f6f8f5c --- /dev/null +++ b/changes/ce/perf-11490.en.md @@ -0,0 +1 @@ +Quickly return the result when the password is absent in password-based authentication. From 45e2e687e5b3dc5f6d99dbbce000e3375ced6f81 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 22 Aug 2023 10:16:03 -0300 Subject: [PATCH 033/119] fix(greptimedb_bridge): avoid double-parsing write syntax during probe and remove code duplication Fixes https://emqx.atlassian.net/browse/EMQX-10843 --- .../src/emqx_bridge_greptimedb.app.src | 2 +- .../src/emqx_bridge_greptimedb.erl | 162 +------- .../src/emqx_bridge_greptimedb_connector.erl | 4 +- .../test/emqx_bridge_greptimedb_SUITE.erl | 94 +++++ .../test/emqx_bridge_greptimedb_tests.erl | 348 ------------------ changes/ee/fix-11492.en.md | 1 + rebar.config.erl | 1 + 7 files changed, 102 insertions(+), 510 deletions(-) delete mode 100644 apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_tests.erl create mode 100644 changes/ee/fix-11492.en.md diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src index fa7e0d4af..c3e3d34e2 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_greptimedb, [ {description, "EMQX GreptimeDB Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl index 877e464dd..d63103e2e 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl @@ -21,11 +21,6 @@ desc/1 ]). --type write_syntax() :: list(). --reflect_type([write_syntax/0]). --typerefl_from_string({write_syntax/0, ?MODULE, to_influx_lines}). --export([to_influx_lines/1]). - %% ------------------------------------------------------------------------------------------------- %% api @@ -131,169 +126,16 @@ desc(_) -> undefined. write_syntax(type) -> - ?MODULE:write_syntax(); + emqx_bridge_influxdb:write_syntax(); write_syntax(required) -> true; write_syntax(validator) -> [?NOT_EMPTY("the value of the field 'write_syntax' cannot be empty")]; write_syntax(converter) -> - fun to_influx_lines/1; + fun emqx_bridge_influxdb:to_influx_lines/1; write_syntax(desc) -> ?DESC("write_syntax"); write_syntax(format) -> <<"sql">>; write_syntax(_) -> undefined. - -to_influx_lines(RawLines) -> - try - influx_lines(str(RawLines), []) - catch - _:Reason:Stacktrace -> - Msg = lists:flatten( - io_lib:format("Unable to parse Greptimedb line protocol: ~p", [RawLines]) - ), - ?SLOG(error, #{msg => Msg, error_reason => Reason, stacktrace => Stacktrace}), - throw(Msg) - end. - --define(MEASUREMENT_ESC_CHARS, [$,, $\s]). --define(TAG_FIELD_KEY_ESC_CHARS, [$,, $=, $\s]). --define(FIELD_VAL_ESC_CHARS, [$", $\\]). -% Common separator for both tags and fields --define(SEP, $\s). --define(MEASUREMENT_TAG_SEP, $,). --define(KEY_SEP, $=). --define(VAL_SEP, $,). --define(NON_EMPTY, [_ | _]). - -influx_lines([] = _RawLines, Acc) -> - ?NON_EMPTY = lists:reverse(Acc); -influx_lines(RawLines, Acc) -> - {Acc1, RawLines1} = influx_line(string:trim(RawLines, leading, "\s\n"), Acc), - influx_lines(RawLines1, Acc1). - -influx_line([], Acc) -> - {Acc, []}; -influx_line(Line, Acc) -> - {?NON_EMPTY = Measurement, Line1} = measurement(Line), - {Tags, Line2} = tags(Line1), - {?NON_EMPTY = Fields, Line3} = influx_fields(Line2), - {Timestamp, Line4} = timestamp(Line3), - { - [ - #{ - measurement => Measurement, - tags => Tags, - fields => Fields, - timestamp => Timestamp - } - | Acc - ], - Line4 - }. - -measurement(Line) -> - unescape(?MEASUREMENT_ESC_CHARS, [?MEASUREMENT_TAG_SEP, ?SEP], Line, []). - -tags([?MEASUREMENT_TAG_SEP | Line]) -> - tags1(Line, []); -tags(Line) -> - {[], Line}. - -%% Empty line is invalid as fields are required after tags, -%% need to break recursion here and fail later on parsing fields -tags1([] = Line, Acc) -> - {lists:reverse(Acc), Line}; -%% Matching non empty Acc treats lines like "m, field=field_val" invalid -tags1([?SEP | _] = Line, ?NON_EMPTY = Acc) -> - {lists:reverse(Acc), Line}; -tags1(Line, Acc) -> - {Tag, Line1} = tag(Line), - tags1(Line1, [Tag | Acc]). - -tag(Line) -> - {?NON_EMPTY = Key, Line1} = key(Line), - {?NON_EMPTY = Val, Line2} = tag_val(Line1), - {{Key, Val}, Line2}. - -tag_val(Line) -> - {Val, Line1} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?VAL_SEP, ?SEP], Line, []), - {Val, strip_l(Line1, ?VAL_SEP)}. - -influx_fields([?SEP | Line]) -> - fields1(string:trim(Line, leading, "\s"), []). - -%% Timestamp is optional, so fields may be at the very end of the line -fields1([Ch | _] = Line, Acc) when Ch =:= ?SEP; Ch =:= $\n -> - {lists:reverse(Acc), Line}; -fields1([] = Line, Acc) -> - {lists:reverse(Acc), Line}; -fields1(Line, Acc) -> - {Field, Line1} = field(Line), - fields1(Line1, [Field | Acc]). - -field(Line) -> - {?NON_EMPTY = Key, Line1} = key(Line), - {Val, Line2} = field_val(Line1), - {{Key, Val}, Line2}. - -field_val([$" | Line]) -> - {Val, [$" | Line1]} = unescape(?FIELD_VAL_ESC_CHARS, [$"], Line, []), - %% Quoted val can be empty - {Val, strip_l(Line1, ?VAL_SEP)}; -field_val(Line) -> - %% Unquoted value should not be un-escaped according to Greptimedb protocol, - %% as it can only hold float, integer, uinteger or boolean value. - %% However, as templates are possible, un-escaping is applied here, - %% which also helps to detect some invalid lines, e.g.: "m,tag=1 field= ${timestamp}" - {Val, Line1} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?VAL_SEP, ?SEP, $\n], Line, []), - {?NON_EMPTY = Val, strip_l(Line1, ?VAL_SEP)}. - -timestamp([?SEP | Line]) -> - Line1 = string:trim(Line, leading, "\s"), - %% Similarly to unquoted field value, un-escape a timestamp to validate and handle - %% potentially escaped characters in a template - {T, Line2} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?SEP, $\n], Line1, []), - {timestamp1(T), Line2}; -timestamp(Line) -> - {undefined, Line}. - -timestamp1(?NON_EMPTY = Ts) -> Ts; -timestamp1(_Ts) -> undefined. - -%% Common for both tag and field keys -key(Line) -> - {Key, Line1} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?KEY_SEP], Line, []), - {Key, strip_l(Line1, ?KEY_SEP)}. - -%% Only strip a character between pairs, don't strip it(and let it fail) -%% if the char to be stripped is at the end, e.g.: m,tag=val, field=val -strip_l([Ch, Ch1 | Str], Ch) when Ch1 =/= ?SEP -> - [Ch1 | Str]; -strip_l(Str, _Ch) -> - Str. - -unescape(EscapeChars, SepChars, [$\\, Char | T], Acc) -> - ShouldEscapeBackslash = lists:member($\\, EscapeChars), - Acc1 = - case lists:member(Char, EscapeChars) of - true -> [Char | Acc]; - false when not ShouldEscapeBackslash -> [Char, $\\ | Acc] - end, - unescape(EscapeChars, SepChars, T, Acc1); -unescape(EscapeChars, SepChars, [Char | T] = L, Acc) -> - IsEscapeChar = lists:member(Char, EscapeChars), - case lists:member(Char, SepChars) of - true -> {lists:reverse(Acc), L}; - false when not IsEscapeChar -> unescape(EscapeChars, SepChars, T, [Char | Acc]) - end; -unescape(_EscapeChars, _SepChars, [] = L, Acc) -> - {lists:reverse(Acc), L}. - -str(A) when is_atom(A) -> - atom_to_list(A); -str(B) when is_binary(B) -> - binary_to_list(B); -str(S) when is_list(S) -> - S. diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index 89fad78d2..d474db58c 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -68,7 +68,9 @@ on_start(InstId, Config) -> on_stop(InstId, _State) -> case emqx_resource:get_allocated_resources(InstId) of #{?greptime_client := Client} -> - greptimedb:stop_client(Client); + Res = greptimedb:stop_client(Client), + ?tp(greptimedb_client_stopped, #{instance_id => InstId}), + Res; _ -> ok end. diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl index d4bc5b01e..15133a1ad 100644 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl @@ -112,6 +112,9 @@ init_per_group(GreptimedbType, Config0) when {proxy_host, ProxyHost}, {proxy_port, ProxyPort}, {proxy_name, ProxyName}, + {bridge_type, greptimedb}, + {bridge_name, Name}, + {bridge_config, GreptimedbConfig}, {greptimedb_host, GreptimedbHost}, {greptimedb_port, GreptimedbPort}, {greptimedb_http_port, GreptimedbHttpPort}, @@ -457,6 +460,97 @@ t_start_ok(Config) -> ), ok. +t_start_stop(Config) -> + %% we can't use this test case directly because `greptimedb_worker' apparently leaks + %% atoms... + %% ok = emqx_bridge_testlib:t_start_stop(Config, greptimedb_client_stopped), + BridgeType = ?config(bridge_type, Config), + BridgeName = ?config(bridge_name, Config), + BridgeConfig = ?config(bridge_config, Config), + StopTracePoint = greptimedb_client_stopped, + ResourceId = emqx_bridge_resource:resource_id(BridgeType, BridgeName), + ?check_trace( + begin + ProbeRes0 = emqx_bridge_testlib:probe_bridge_api( + BridgeType, + BridgeName, + BridgeConfig + ), + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes0), + ?assertMatch({ok, _}, emqx_bridge:create(BridgeType, BridgeName, BridgeConfig)), + + %% Since the connection process is async, we give it some time to + %% stabilize and avoid flakiness. + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + + %% `start` bridge to trigger `already_started` + ?assertMatch( + {ok, {{_, 204, _}, _Headers, []}}, + emqx_bridge_testlib:op_bridge_api("start", BridgeType, BridgeName) + ), + + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)), + + ?assertMatch( + {{ok, _}, {ok, _}}, + ?wait_async_action( + emqx_bridge_testlib:op_bridge_api("stop", BridgeType, BridgeName), + #{?snk_kind := StopTracePoint}, + 5_000 + ) + ), + + ?assertEqual( + {error, resource_is_stopped}, emqx_resource_manager:health_check(ResourceId) + ), + + ?assertMatch( + {ok, {{_, 204, _}, _Headers, []}}, + emqx_bridge_testlib:op_bridge_api("stop", BridgeType, BridgeName) + ), + + ?assertEqual( + {error, resource_is_stopped}, emqx_resource_manager:health_check(ResourceId) + ), + + ?assertMatch( + {ok, {{_, 204, _}, _Headers, []}}, + emqx_bridge_testlib:op_bridge_api("start", BridgeType, BridgeName) + ), + + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + + %% Disable the bridge, which will also stop it. + ?assertMatch( + {{ok, _}, {ok, _}}, + ?wait_async_action( + emqx_bridge:disable_enable(disable, BridgeType, BridgeName), + #{?snk_kind := StopTracePoint}, + 5_000 + ) + ), + + ok + end, + fun(Trace) -> + %% one for probe, two for real + ?assertMatch( + [_, #{instance_id := ResourceId}, #{instance_id := ResourceId}], + ?of_kind(StopTracePoint, Trace) + ), + ok + end + ), + ok. + t_start_already_started(Config) -> Type = greptimedb_type_bin(?config(greptimedb_type, Config)), Name = ?config(greptimedb_name, Config), diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_tests.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_tests.erl deleted file mode 100644 index a07ccd92d..000000000 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_tests.erl +++ /dev/null @@ -1,348 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- --module(emqx_bridge_greptimedb_tests). - --include_lib("eunit/include/eunit.hrl"). - --define(INVALID_LINES, [ - " ", - " \n", - " \n\n\n ", - "\n", - " \n\n \n \n", - "measurement", - "measurement ", - "measurement,tag", - "measurement field", - "measurement,tag field", - "measurement,tag field ${timestamp}", - "measurement,tag=", - "measurement,tag=tag1", - "measurement,tag =", - "measurement field=", - "measurement field= ", - "measurement field = ", - "measurement, tag = field = ", - "measurement, tag = field = ", - "measurement, tag = tag_val field = field_val", - "measurement, tag = tag_val field = field_val ${timestamp}", - "measurement,= = ${timestamp}", - "measurement,t=a, f=a, ${timestamp}", - "measurement,t=a,t1=b, f=a,f1=b, ${timestamp}", - "measurement,t=a,t1=b, f=a,f1=b,", - "measurement,t=a, t1=b, f=a,f1=b,", - "measurement,t=a,,t1=b, f=a,f1=b,", - "measurement,t=a,,t1=b f=a,,f1=b", - "measurement,t=a,,t1=b f=a,f1=b ${timestamp}", - "measurement, f=a,f1=b", - "measurement, f=a,f1=b ${timestamp}", - "measurement,, f=a,f1=b ${timestamp}", - "measurement,, f=a,f1=b", - "measurement,, f=a,f1=b,, ${timestamp}", - "measurement f=a,f1=b,, ${timestamp}", - "measurement,t=a f=a,f1=b,, ${timestamp}", - "measurement,t=a f=a,f1=b,, ", - "measurement,t=a f=a,f1=b,,", - "measurement, t=a f=a,f1=b", - "measurement,t=a f=a, f1=b", - "measurement,t=a f=a, f1=b ${timestamp}", - "measurement, t=a f=a, f1=b ${timestamp}", - "measurement,t= a f=a,f1=b ${timestamp}", - "measurement,t= a f=a,f1 =b ${timestamp}", - "measurement, t = a f = a,f1 = b ${timestamp}", - "measurement,t=a f=a,f1=b \n ${timestamp}", - "measurement,t=a \n f=a,f1=b \n ${timestamp}", - "measurement,t=a \n f=a,f1=b \n ", - "\n measurement,t=a \n f=a,f1=b \n ${timestamp}", - "\n measurement,t=a \n f=a,f1=b \n", - %% not escaped backslash in a quoted field value is invalid - "measurement,tag=1 field=\"val\\1\"" -]). - --define(VALID_LINE_PARSED_PAIRS, [ - {"m1,tag=tag1 field=field1 ${timestamp1}", #{ - measurement => "m1", - tags => [{"tag", "tag1"}], - fields => [{"field", "field1"}], - timestamp => "${timestamp1}" - }}, - {"m2,tag=tag2 field=field2", #{ - measurement => "m2", - tags => [{"tag", "tag2"}], - fields => [{"field", "field2"}], - timestamp => undefined - }}, - {"m3 field=field3 ${timestamp3}", #{ - measurement => "m3", - tags => [], - fields => [{"field", "field3"}], - timestamp => "${timestamp3}" - }}, - {"m4 field=field4", #{ - measurement => "m4", - tags => [], - fields => [{"field", "field4"}], - timestamp => undefined - }}, - {"m5,tag=tag5,tag_a=tag5a,tag_b=tag5b field=field5,field_a=field5a,field_b=field5b ${timestamp5}", - #{ - measurement => "m5", - tags => [{"tag", "tag5"}, {"tag_a", "tag5a"}, {"tag_b", "tag5b"}], - fields => [{"field", "field5"}, {"field_a", "field5a"}, {"field_b", "field5b"}], - timestamp => "${timestamp5}" - }}, - {"m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=field6,field_a=field6a,field_b=field6b", #{ - measurement => "m6", - tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}], - fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], - timestamp => undefined - }}, - {"m7,tag=tag7,tag_a=\"tag7a\",tag_b=tag7b field=\"field7\",field_a=field7a,field_b=\"field7b\"", - #{ - measurement => "m7", - tags => [{"tag", "tag7"}, {"tag_a", "\"tag7a\""}, {"tag_b", "tag7b"}], - fields => [{"field", "field7"}, {"field_a", "field7a"}, {"field_b", "field7b"}], - timestamp => undefined - }}, - {"m8,tag=tag8,tag_a=\"tag8a\",tag_b=tag8b field=\"field8\",field_a=field8a,field_b=\"field8b\" ${timestamp8}", - #{ - measurement => "m8", - tags => [{"tag", "tag8"}, {"tag_a", "\"tag8a\""}, {"tag_b", "tag8b"}], - fields => [{"field", "field8"}, {"field_a", "field8a"}, {"field_b", "field8b"}], - timestamp => "${timestamp8}" - }}, - {"m9,tag=tag9,tag_a=\"tag9a\",tag_b=tag9b field=\"field9\",field_a=field9a,field_b=\"\" ${timestamp9}", - #{ - measurement => "m9", - tags => [{"tag", "tag9"}, {"tag_a", "\"tag9a\""}, {"tag_b", "tag9b"}], - fields => [{"field", "field9"}, {"field_a", "field9a"}, {"field_b", ""}], - timestamp => "${timestamp9}" - }}, - {"m10 field=\"\" ${timestamp10}", #{ - measurement => "m10", - tags => [], - fields => [{"field", ""}], - timestamp => "${timestamp10}" - }} -]). - --define(VALID_LINE_EXTRA_SPACES_PARSED_PAIRS, [ - {"\n m1,tag=tag1 field=field1 ${timestamp1} \n", #{ - measurement => "m1", - tags => [{"tag", "tag1"}], - fields => [{"field", "field1"}], - timestamp => "${timestamp1}" - }}, - {" m2,tag=tag2 field=field2 ", #{ - measurement => "m2", - tags => [{"tag", "tag2"}], - fields => [{"field", "field2"}], - timestamp => undefined - }}, - {" m3 field=field3 ${timestamp3} ", #{ - measurement => "m3", - tags => [], - fields => [{"field", "field3"}], - timestamp => "${timestamp3}" - }}, - {" \n m4 field=field4\n ", #{ - measurement => "m4", - tags => [], - fields => [{"field", "field4"}], - timestamp => undefined - }}, - {" \n m5,tag=tag5,tag_a=tag5a,tag_b=tag5b field=field5,field_a=field5a,field_b=field5b ${timestamp5} \n", - #{ - measurement => "m5", - tags => [{"tag", "tag5"}, {"tag_a", "tag5a"}, {"tag_b", "tag5b"}], - fields => [{"field", "field5"}, {"field_a", "field5a"}, {"field_b", "field5b"}], - timestamp => "${timestamp5}" - }}, - {" m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=field6,field_a=field6a,field_b=field6b\n ", #{ - measurement => "m6", - tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}], - fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], - timestamp => undefined - }} -]). - --define(VALID_LINE_PARSED_ESCAPED_CHARS_PAIRS, [ - {"m\\ =1\\,,\\,tag\\ \\==\\=tag\\ 1\\, \\,fie\\ ld\\ =\\ field\\,1 ${timestamp1}", #{ - measurement => "m =1,", - tags => [{",tag =", "=tag 1,"}], - fields => [{",fie ld ", " field,1"}], - timestamp => "${timestamp1}" - }}, - {"m2,tag=tag2 field=\"field \\\"2\\\",\n\"", #{ - measurement => "m2", - tags => [{"tag", "tag2"}], - fields => [{"field", "field \"2\",\n"}], - timestamp => undefined - }}, - {"m\\ 3 field=\"field3\" ${payload.timestamp\\ 3}", #{ - measurement => "m 3", - tags => [], - fields => [{"field", "field3"}], - timestamp => "${payload.timestamp 3}" - }}, - {"m4 field=\"\\\"field\\\\4\\\"\"", #{ - measurement => "m4", - tags => [], - fields => [{"field", "\"field\\4\""}], - timestamp => undefined - }}, - { - "m5\\,mA,tag=\\=tag5\\=,\\,tag_a\\,=tag\\ 5a,tag_b=tag5b \\ field\\ =field5," - "field\\ _\\ a=field5a,\\,field_b\\ =\\=\\,\\ field5b ${timestamp5}", - #{ - measurement => "m5,mA", - tags => [{"tag", "=tag5="}, {",tag_a,", "tag 5a"}, {"tag_b", "tag5b"}], - fields => [ - {" field ", "field5"}, {"field _ a", "field5a"}, {",field_b ", "=, field5b"} - ], - timestamp => "${timestamp5}" - } - }, - {"m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=\"field6\",field_a=\"field6a\",field_b=\"field6b\"", - #{ - measurement => "m6", - tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}], - fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], - timestamp => undefined - }}, - { - "\\ \\ m7\\ \\ ,tag=\\ tag\\,7\\ ,tag_a=\"tag7a\",tag_b\\,tag1=tag7b field=\"field7\"," - "field_a=field7a,field_b=\"field7b\\\\\n\"", - #{ - measurement => " m7 ", - tags => [{"tag", " tag,7 "}, {"tag_a", "\"tag7a\""}, {"tag_b,tag1", "tag7b"}], - fields => [{"field", "field7"}, {"field_a", "field7a"}, {"field_b", "field7b\\\n"}], - timestamp => undefined - } - }, - { - "m8,tag=tag8,tag_a=\"tag8a\",tag_b=tag8b field=\"field8\",field_a=field8a," - "field_b=\"\\\"field\\\" = 8b\" ${timestamp8}", - #{ - measurement => "m8", - tags => [{"tag", "tag8"}, {"tag_a", "\"tag8a\""}, {"tag_b", "tag8b"}], - fields => [{"field", "field8"}, {"field_a", "field8a"}, {"field_b", "\"field\" = 8b"}], - timestamp => "${timestamp8}" - } - }, - {"m\\9,tag=tag9,tag_a=\"tag9a\",tag_b=tag9b field\\=field=\"field9\",field_a=field9a,field_b=\"\" ${timestamp9}", - #{ - measurement => "m\\9", - tags => [{"tag", "tag9"}, {"tag_a", "\"tag9a\""}, {"tag_b", "tag9b"}], - fields => [{"field=field", "field9"}, {"field_a", "field9a"}, {"field_b", ""}], - timestamp => "${timestamp9}" - }}, - {"m\\,10 \"field\\\\\"=\"\" ${timestamp10}", #{ - measurement => "m,10", - tags => [], - %% backslash should not be un-escaped in tag key - fields => [{"\"field\\\\\"", ""}], - timestamp => "${timestamp10}" - }} -]). - --define(VALID_LINE_PARSED_ESCAPED_CHARS_EXTRA_SPACES_PAIRS, [ - {" \n m\\ =1\\,,\\,tag\\ \\==\\=tag\\ 1\\, \\,fie\\ ld\\ =\\ field\\,1 ${timestamp1} ", #{ - measurement => "m =1,", - tags => [{",tag =", "=tag 1,"}], - fields => [{",fie ld ", " field,1"}], - timestamp => "${timestamp1}" - }}, - {" m2,tag=tag2 field=\"field \\\"2\\\",\n\" ", #{ - measurement => "m2", - tags => [{"tag", "tag2"}], - fields => [{"field", "field \"2\",\n"}], - timestamp => undefined - }}, - {" m\\ 3 field=\"field3\" ${payload.timestamp\\ 3} ", #{ - measurement => "m 3", - tags => [], - fields => [{"field", "field3"}], - timestamp => "${payload.timestamp 3}" - }}, - {" m4 field=\"\\\"field\\\\4\\\"\" ", #{ - measurement => "m4", - tags => [], - fields => [{"field", "\"field\\4\""}], - timestamp => undefined - }}, - { - " m5\\,mA,tag=\\=tag5\\=,\\,tag_a\\,=tag\\ 5a,tag_b=tag5b \\ field\\ =field5," - "field\\ _\\ a=field5a,\\,field_b\\ =\\=\\,\\ field5b ${timestamp5} ", - #{ - measurement => "m5,mA", - tags => [{"tag", "=tag5="}, {",tag_a,", "tag 5a"}, {"tag_b", "tag5b"}], - fields => [ - {" field ", "field5"}, {"field _ a", "field5a"}, {",field_b ", "=, field5b"} - ], - timestamp => "${timestamp5}" - } - }, - {" m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=\"field6\",field_a=\"field6a\",field_b=\"field6b\" ", - #{ - measurement => "m6", - tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}], - fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], - timestamp => undefined - }} -]). - -invalid_write_syntax_line_test_() -> - [?_assertThrow(_, to_influx_lines(L)) || L <- ?INVALID_LINES]. - -invalid_write_syntax_multiline_test_() -> - LinesList = [ - join("\n", ?INVALID_LINES), - join("\n\n\n", ?INVALID_LINES), - join("\n\n", lists:reverse(?INVALID_LINES)) - ], - [?_assertThrow(_, to_influx_lines(Lines)) || Lines <- LinesList]. - -valid_write_syntax_test_() -> - test_pairs(?VALID_LINE_PARSED_PAIRS). - -valid_write_syntax_with_extra_spaces_test_() -> - test_pairs(?VALID_LINE_EXTRA_SPACES_PARSED_PAIRS). - -valid_write_syntax_escaped_chars_test_() -> - test_pairs(?VALID_LINE_PARSED_ESCAPED_CHARS_PAIRS). - -valid_write_syntax_escaped_chars_with_extra_spaces_test_() -> - test_pairs(?VALID_LINE_PARSED_ESCAPED_CHARS_EXTRA_SPACES_PAIRS). - -test_pairs(PairsList) -> - {Lines, AllExpected} = lists:unzip(PairsList), - JoinedLines = join("\n", Lines), - JoinedLines1 = join("\n\n\n", Lines), - JoinedLines2 = join("\n\n", lists:reverse(Lines)), - SingleLineTests = - [ - ?_assertEqual([Expected], to_influx_lines(Line)) - || {Line, Expected} <- PairsList - ], - JoinedLinesTests = - [ - ?_assertEqual(AllExpected, to_influx_lines(JoinedLines)), - ?_assertEqual(AllExpected, to_influx_lines(JoinedLines1)), - ?_assertEqual(lists:reverse(AllExpected), to_influx_lines(JoinedLines2)) - ], - SingleLineTests ++ JoinedLinesTests. - -join(Sep, LinesList) -> - lists:flatten(lists:join(Sep, LinesList)). - -to_influx_lines(RawLines) -> - OldLevel = emqx_logger:get_primary_log_level(), - try - %% mute error logs from this call - emqx_logger:set_primary_log_level(none), - emqx_bridge_greptimedb:to_influx_lines(RawLines) - after - emqx_logger:set_primary_log_level(OldLevel) - end. diff --git a/changes/ee/fix-11492.en.md b/changes/ee/fix-11492.en.md new file mode 100644 index 000000000..9f61abee2 --- /dev/null +++ b/changes/ee/fix-11492.en.md @@ -0,0 +1 @@ +Fixed an issue which would yield false negatives when testing the connectivity of GreptimeDB bridges. diff --git a/rebar.config.erl b/rebar.config.erl index ad6f425a0..02e946e15 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -84,6 +84,7 @@ is_community_umbrella_app("apps/emqx_bridge_cassandra") -> false; is_community_umbrella_app("apps/emqx_bridge_opents") -> false; is_community_umbrella_app("apps/emqx_bridge_clickhouse") -> false; is_community_umbrella_app("apps/emqx_bridge_dynamo") -> false; +is_community_umbrella_app("apps/emqx_bridge_greptimedb") -> false; is_community_umbrella_app("apps/emqx_bridge_hstreamdb") -> false; is_community_umbrella_app("apps/emqx_bridge_influxdb") -> false; is_community_umbrella_app("apps/emqx_bridge_iotdb") -> false; From 20b46091f928c4a6e0f364390624f343c0cb8c44 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Mon, 21 Aug 2023 21:51:46 -0300 Subject: [PATCH 034/119] fix(kinesis): set validators for static constraints Fixes https://emqx.atlassian.net/browse/EMQX-10833 --- .../src/emqx_bridge_kinesis.erl | 27 +++++++++++++++++-- ...mqx_bridge_kinesis_impl_producer_SUITE.erl | 14 ++++++++++ changes/ee/fix-11494.en.md | 1 + 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 changes/ee/fix-11494.en.md diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl index 92faa2279..d98e7ab11 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl @@ -30,8 +30,27 @@ roots() -> fields("config_producer") -> emqx_bridge_schema:common_bridge_fields() ++ - emqx_resource_schema:fields("resource_opts") ++ - fields(connector_config) ++ fields(producer); + fields("resource_opts") ++ + fields(connector_config) ++ + fields(producer); +fields("resource_opts") -> + [ + {resource_opts, + mk( + ref(?MODULE, "creation_opts"), + #{ + required => false, + default => #{}, + desc => ?DESC(emqx_resource_schema, "creation_opts") + } + )} + ]; +fields("creation_opts") -> + emqx_resource_schema:create_opts([ + {batch_size, #{ + validator => emqx_resource_validator:max(int, 500) + }} + ]); fields(connector_config) -> [ {aws_access_key_id, @@ -121,6 +140,8 @@ fields("put_producer") -> desc("config_producer") -> ?DESC("desc_config"); +desc("creation_opts") -> + ?DESC(emqx_resource_schema, "creation_opts"); desc(_) -> undefined. @@ -161,6 +182,8 @@ mk(Type, Meta) -> hoconsc:mk(Type, Meta). enum(OfSymbols) -> hoconsc:enum(OfSymbols). +ref(Module, Name) -> hoconsc:ref(Module, Name). + type_field_producer() -> {type, mk(enum([kinesis_producer]), #{required => true, desc => ?DESC("desc_type")})}. diff --git a/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl b/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl index 69b4a2d86..ea926fc33 100644 --- a/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl @@ -902,3 +902,17 @@ t_empty_payload_template(Config) -> emqx_utils_json:decode(Data, [return_maps]) ), ok. + +t_validate_static_constraints(Config) -> + % From : + % "Each PutRecords request can support up to 500 records. + % Each record in the request can be as large as 1 MiB, + % up to a limit of 5 MiB for the entire request, including partition keys." + % + % Message size and request size shall be controlled by user, so there is no validators + % for them - if exceeded, it will fail like on `t_publish_big_msg` test. + ?assertThrow( + {emqx_bridge_schema, [#{kind := validation_error, value := 501}]}, + generate_config([{batch_size, 501} | Config]) + ), + ok. diff --git a/changes/ee/fix-11494.en.md b/changes/ee/fix-11494.en.md new file mode 100644 index 000000000..5ff887562 --- /dev/null +++ b/changes/ee/fix-11494.en.md @@ -0,0 +1 @@ +Added schema validator to reflect Amazon Kinesis' static constraint: batch request can support up to 500 records (max batch size); From ae10415fc35adad98c2a179bc26b442f37581f13 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 23 Aug 2023 09:00:41 +0800 Subject: [PATCH 035/119] feat: disabled the Erlang VM Prometheus exporter by default to improve performance and security --- apps/emqx/src/emqx_metrics.erl | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 50 +++++++++++++------ .../src/emqx_prometheus.app.src | 2 +- .../src/emqx_prometheus_config.erl | 21 ++++++-- .../src/emqx_prometheus_schema.erl | 12 ++--- changes/ce/feat-11496.en.md | 1 + 6 files changed, 62 insertions(+), 26 deletions(-) create mode 100644 changes/ce/feat-11496.en.md diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index 21a114c0f..6b46af257 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -267,7 +267,7 @@ {counter, 'authentication.failure'} ]). -%% Overload protetion counters +%% Overload protection counters -define(OLP_METRICS, [ {counter, 'olp.delay.ok'}, {counter, 'olp.delay.timeout'}, diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 5b73b9e03..94cdd2ecd 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1105,12 +1105,7 @@ translation("gen_rpc") -> [{"default_client_driver", fun tr_default_config_driver/1}]; translation("prometheus") -> [ - {"vm_dist_collector_metrics", fun tr_vm_dist_collector/1}, - {"mnesia_collector_metrics", fun tr_mnesia_collector/1}, - {"vm_statistics_collector_metrics", fun tr_vm_statistics_collector/1}, - {"vm_system_info_collector_metrics", fun tr_vm_system_info_collector/1}, - {"vm_memory_collector_metrics", fun tr_vm_memory_collector/1}, - {"vm_msacc_collector_metrics", fun tr_vm_msacc_collector/1} + {"collectors", fun tr_prometheus_collectors/1} ]; translation("vm_args") -> [ @@ -1120,26 +1115,53 @@ translation("vm_args") -> tr_vm_args_process_limit(Conf) -> 2 * conf_get("node.max_ports", Conf, ?DEFAULT_MAX_PORTS). +tr_prometheus_collectors(Conf) -> + [ + %% builtin collectors + prometheus_boolean, + prometheus_counter, + prometheus_gauge, + prometheus_histogram, + prometheus_quantile_summary, + prometheus_summary, + %% emqx collectors + emqx_prometheus, + emqx_prometheus_mria + %% builtin vm collectors + | tr_vm_dist_collector(Conf) ++ + tr_mnesia_collector(Conf) ++ + tr_vm_statistics_collector(Conf) ++ + tr_vm_system_info_collector(Conf) ++ + tr_vm_memory_collector(Conf) ++ + tr_vm_msacc_collector(Conf) + ]. + tr_vm_dist_collector(Conf) -> - metrics_enabled(conf_get("prometheus.vm_dist_collector", Conf, enabled)). + Enabled = conf_get("prometheus.vm_dist_collector", Conf, disabled), + collector_enabled(Enabled, prometheus_vm_dist_collector). tr_mnesia_collector(Conf) -> - metrics_enabled(conf_get("prometheus.mnesia_collector", Conf, enabled)). + Enabled = conf_get("prometheus.mnesia_collector", Conf, disabled), + collector_enabled(Enabled, prometheus_mnesia_collector). tr_vm_statistics_collector(Conf) -> - metrics_enabled(conf_get("prometheus.vm_statistics_collector", Conf, enabled)). + Enabled = conf_get("prometheus.vm_statistics_collector", Conf, disabled), + collector_enabled(Enabled, prometheus_vm_statistics_collector). tr_vm_system_info_collector(Conf) -> - metrics_enabled(conf_get("prometheus.vm_system_info_collector", Conf, enabled)). + Enabled = conf_get("prometheus.vm_system_info_collector", Conf, disabled), + collector_enabled(Enabled, prometheus_vm_system_info_collector). tr_vm_memory_collector(Conf) -> - metrics_enabled(conf_get("prometheus.vm_memory_collector", Conf, enabled)). + Enabled = conf_get("prometheus.vm_memory_collector", Conf, disabled), + collector_enabled(Enabled, prometheus_vm_memory_collector). tr_vm_msacc_collector(Conf) -> - metrics_enabled(conf_get("prometheus.vm_msacc_collector", Conf, enabled)). + Enabled = conf_get("prometheus.vm_msacc_collector", Conf, disabled), + collector_enabled(Enabled, prometheus_vm_msacc_collector). -metrics_enabled(enabled) -> all; -metrics_enabled(disabled) -> []. +collector_enabled(enabled, Collector) -> [Collector]; +collector_enabled(disabled, _) -> []. tr_default_config_driver(Conf) -> conf_get("rpc.driver", Conf). diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index 10fd75e98..c4abbec27 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.15"}, + {vsn, "5.0.16"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus_config.erl b/apps/emqx_prometheus/src/emqx_prometheus_config.erl index 39d2d4f6b..abd3427d3 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_config.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_config.erl @@ -46,12 +46,25 @@ remove_handler() -> ok. post_config_update(?PROMETHEUS, _Req, New, _Old, AppEnvs) -> - application:set_env(AppEnvs), - update_prometheus(New); + update_prometheus(AppEnvs), + update_push_gateway(New); post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> ok. -update_prometheus(#{enable := true}) -> +update_prometheus(AppEnvs) -> + {ok, PrevCollectors} = application:get_env(prometheus, collectors), + CurCollectors = proplists:get_value(collectors, proplists:get_value(prometheus, AppEnvs)), + lists:foreach( + fun prometheus_registry:deregister_collector/1, + PrevCollectors -- CurCollectors + ), + lists:foreach( + fun prometheus_registry:register_collector/1, + CurCollectors -- PrevCollectors + ), + application:set_env(AppEnvs). + +update_push_gateway(#{enable := true}) -> emqx_prometheus_sup:start_child(?APP); -update_prometheus(#{enable := false}) -> +update_push_gateway(#{enable := false}) -> emqx_prometheus_sup:stop_child(?APP). diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 3884f7065..41d74aa20 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -99,7 +99,7 @@ fields("prometheus") -> ?HOCON( hoconsc:enum([enabled, disabled]), #{ - default => enabled, + default => disabled, required => true, importance => ?IMPORTANCE_LOW, desc => ?DESC(mnesia_collector) @@ -110,7 +110,7 @@ fields("prometheus") -> ?HOCON( hoconsc:enum([enabled, disabled]), #{ - default => enabled, + default => disabled, required => true, importance => ?IMPORTANCE_LOW, desc => ?DESC(vm_statistics_collector) @@ -121,7 +121,7 @@ fields("prometheus") -> ?HOCON( hoconsc:enum([enabled, disabled]), #{ - default => enabled, + default => disabled, required => true, importance => ?IMPORTANCE_LOW, desc => ?DESC(vm_system_info_collector) @@ -133,7 +133,7 @@ fields("prometheus") -> ?HOCON( hoconsc:enum([enabled, disabled]), #{ - default => enabled, + default => disabled, required => true, importance => ?IMPORTANCE_LOW, desc => ?DESC(vm_memory_collector) @@ -144,7 +144,7 @@ fields("prometheus") -> ?HOCON( hoconsc:enum([enabled, disabled]), #{ - default => enabled, + default => disabled, required => true, importance => ?IMPORTANCE_LOW, desc => ?DESC(vm_msacc_collector) @@ -178,5 +178,5 @@ validate_push_gateway_server(Url) -> translation(Name) -> %% translate 'vm_dist_collector', 'mnesia_collector', 'vm_statistics_collector', %% 'vm_system_info_collector', 'vm_memory_collector', 'vm_msacc_collector' - %% to prometheus envrionments + %% to prometheus environments emqx_conf_schema:translation(Name). diff --git a/changes/ce/feat-11496.en.md b/changes/ce/feat-11496.en.md new file mode 100644 index 000000000..5303ce8d8 --- /dev/null +++ b/changes/ce/feat-11496.en.md @@ -0,0 +1 @@ +Disabled the Erlang VM Prometheus exporter by default to improve performance and security. From 188a81913ebd2319dfc90ebbb3403bddd5951b3c Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 23 Aug 2023 10:39:00 +0800 Subject: [PATCH 036/119] test: add prometheus api test --- .../src/emqx_prometheus_config.erl | 2 +- .../test/emqx_prometheus_SUITE.erl | 8 +++---- .../test/emqx_prometheus_api_SUITE.erl | 23 ++++++++++++++++--- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus_config.erl b/apps/emqx_prometheus/src/emqx_prometheus_config.erl index abd3427d3..00dad47f9 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_config.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_config.erl @@ -52,7 +52,7 @@ post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> ok. update_prometheus(AppEnvs) -> - {ok, PrevCollectors} = application:get_env(prometheus, collectors), + PrevCollectors = prometheus_registry:collectors(default), CurCollectors = proplists:get_value(collectors, proplists:get_value(prometheus, AppEnvs)), lists:foreach( fun prometheus_registry:deregister_collector/1, diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl index 77d9902a2..3f9e743f3 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl @@ -30,12 +30,12 @@ " headers = { Authorization = \"some-authz-tokens\"}\n" " job_name = \"${name}~${host}\"\n" " enable = true\n" - " vm_dist_collector = enabled\n" - " mnesia_collector = enabled\n" + " vm_dist_collector = disabled\n" + " mnesia_collector = disabled\n" " vm_statistics_collector = disabled\n" " vm_system_info_collector = disabled\n" - " vm_memory_collector = enabled\n" - " vm_msacc_collector = enabled\n" + " vm_memory_collector = disabled\n" + " vm_msacc_collector = disabled\n" "}\n" >>). diff --git a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl index e29d46720..c7d336554 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl @@ -81,13 +81,30 @@ t_prometheus_api(_) -> ), #{<<"enable">> := Enable} = Conf, ?assertEqual(Enable, undefined =/= erlang:whereis(emqx_prometheus)), - NewConf = Conf#{<<"interval">> => <<"2s">>, <<"vm_statistics_collector">> => <<"disabled">>}, + NewConf = Conf#{<<"interval">> => <<"2s">>, <<"vm_statistics_collector">> => <<"enabled">>}, {ok, Response2} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf), Conf2 = emqx_utils_json:decode(Response2, [return_maps]), ?assertMatch(NewConf, Conf2), - ?assertEqual({ok, []}, application:get_env(prometheus, vm_statistics_collector_metrics)), - ?assertEqual({ok, all}, application:get_env(prometheus, vm_memory_collector_metrics)), + + EnvCollectors = application:get_env(prometheus, collectors, []), + PromCollectors = prometheus_registry:collectors(default), + ?assertEqual(lists:sort(EnvCollectors), lists:sort(PromCollectors)), + ?assert(lists:member(prometheus_vm_statistics_collector, EnvCollectors), EnvCollectors), + + lists:foreach( + fun({C, Enabled}) -> + ?assertEqual(Enabled, lists:member(C, EnvCollectors), EnvCollectors) + end, + [ + {prometheus_vm_dist_collector, false}, + {prometheus_vm_system_info_collector, false}, + {prometheus_vm_memory_collector, false}, + {prometheus_mnesia_collector, false}, + {prometheus_vm_msacc_collector, false}, + {prometheus_vm_statistics_collector, true} + ] + ), NewConf1 = Conf#{<<"enable">> => (not Enable)}, {ok, _Response3} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf1), From df9f8da178bda519ba8d5e05347f45306beef598 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 23 Aug 2023 14:07:37 +0800 Subject: [PATCH 037/119] feat: add overload_protection metrics to prometheus --- .../src/emqx_prometheus.app.src | 2 +- apps/emqx_prometheus/src/emqx_prometheus.erl | 38 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index 10fd75e98..c4abbec27 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.15"}, + {vsn, "5.0.16"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index ac902ca55..29587c24d 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -168,6 +168,7 @@ collect_mf(_Registry, Callback) -> _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_delivery()], _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_client()], _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_session()], + _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_olp()], ok. %% @private @@ -437,6 +438,19 @@ emqx_collect(emqx_session_discarded, Stats) -> emqx_collect(emqx_session_terminated, Stats) -> counter_metric(?C('session.terminated', Stats)); %%-------------------------------------------------------------------- + +%% Metrics - overload protection +emqx_collect(emqx_overload_protection_delay_ok, Stats) -> + counter_metric(?C('olp.delay.ok', Stats)); +emqx_collect(emqx_overload_protection_delay_timeout, Stats) -> + counter_metric(?C('olp.delay.timeout', Stats)); +emqx_collect(emqx_overload_protection_hibernation, Stats) -> + counter_metric(?C('olp.hbn', Stats)); +emqx_collect(emqx_overload_protection_gc, Stats) -> + counter_metric(?C('olp.gc', Stats)); +emqx_collect(emqx_overload_protection_new_conn, Stats) -> + counter_metric(?C('olp.new_conn', Stats)); +%%-------------------------------------------------------------------- %% VM emqx_collect(emqx_vm_cpu_use, VMData) -> @@ -506,6 +520,30 @@ emqx_metrics_packets() -> emqx_packets_auth_sent ]. +emqx_metrics_olp() -> + case is_olp_enabled() of + true -> + [ + emqx_overload_protection_delay_ok, + emqx_overload_protection_delay_timeout, + emqx_overload_protection_hibernation, + emqx_overload_protection_gc, + emqx_overload_protection_new_conn + ]; + false -> + [] + end. + +is_olp_enabled() -> + maps:fold( + fun + (_, #{overload_protection := #{enable := true}}, _Acc) -> true; + (_, _, Acc) -> Acc + end, + false, + emqx_conf:get([zones], #{}) + ). + emqx_metrics_messages() -> [ emqx_messages_received, From 4928c697580b70ccf3d5318a6fc29843bf830671 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 23 Aug 2023 14:11:59 +0800 Subject: [PATCH 038/119] feat: add authorization metrics to prometheus --- apps/emqx_prometheus/src/emqx_prometheus.erl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 29587c24d..bc1564290 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -169,6 +169,7 @@ collect_mf(_Registry, Callback) -> _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_client()], _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_session()], _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_olp()], + _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_acl()], ok. %% @private @@ -451,6 +452,14 @@ emqx_collect(emqx_overload_protection_gc, Stats) -> emqx_collect(emqx_overload_protection_new_conn, Stats) -> counter_metric(?C('olp.new_conn', Stats)); %%-------------------------------------------------------------------- +%% Metrics - acl +emqx_collect(emqx_authorization_allow, Stats) -> + counter_metric(?C('authorization.allow', Stats)); +emqx_collect(emqx_authorization_deny, Stats) -> + counter_metric(?C('authorization.deny', Stats)); +emqx_collect(emqx_authorization_cache_hit, Stats) -> + counter_metric(?C('authorization.cache_hit', Stats)); +%%-------------------------------------------------------------------- %% VM emqx_collect(emqx_vm_cpu_use, VMData) -> @@ -544,6 +553,13 @@ is_olp_enabled() -> emqx_conf:get([zones], #{}) ). +emqx_metrics_acl() -> + [ + emqx_authorization_allow, + emqx_authorization_deny, + emqx_authorization_cache_hit + ]. + emqx_metrics_messages() -> [ emqx_messages_received, From 141676185de921227bec4363155f7ea5e1d44a3b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 23 Aug 2023 14:15:16 +0800 Subject: [PATCH 039/119] feat: add authentication metrics to prometheus --- apps/emqx_prometheus/src/emqx_prometheus.erl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index bc1564290..66ab9f61f 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -170,6 +170,7 @@ collect_mf(_Registry, Callback) -> _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_session()], _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_olp()], _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_acl()], + _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_authn()], ok. %% @private @@ -460,6 +461,14 @@ emqx_collect(emqx_authorization_deny, Stats) -> emqx_collect(emqx_authorization_cache_hit, Stats) -> counter_metric(?C('authorization.cache_hit', Stats)); %%-------------------------------------------------------------------- +%% Metrics - authn +emqx_collect(emqx_authentication_success, Stats) -> + counter_metric(?C('authentication.success', Stats)); +emqx_collect(emqx_authentication_success_anonymous, Stats) -> + counter_metric(?C('authentication.success.anonymous', Stats)); +emqx_collect(emqx_authentication_failure, Stats) -> + counter_metric(?C('authentication.failure', Stats)); +%%-------------------------------------------------------------------- %% VM emqx_collect(emqx_vm_cpu_use, VMData) -> @@ -560,6 +569,13 @@ emqx_metrics_acl() -> emqx_authorization_cache_hit ]. +emqx_metrics_authn() -> + [ + emqx_authentication_success, + emqx_authentication_success_anonymous, + emqx_authentication_failure + ]. + emqx_metrics_messages() -> [ emqx_messages_received, From b9a97923c41777fd0bf26ead131008acbf359edd Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 23 Aug 2023 14:47:53 +0800 Subject: [PATCH 040/119] feat: add some authz/connection missing metrics --- apps/emqx/src/emqx_stats.erl | 4 ++- apps/emqx_opentelemetry/src/emqx_otel.erl | 9 +++++- apps/emqx_prometheus/src/emqx_prometheus.erl | 30 ++++++++++++++++++-- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_stats.erl b/apps/emqx/src/emqx_stats.erl index e590577da..dfd3115f0 100644 --- a/apps/emqx/src/emqx_stats.erl +++ b/apps/emqx/src/emqx_stats.erl @@ -177,7 +177,9 @@ names() -> emqx_subscriptions_shared_count, emqx_subscriptions_shared_max, emqx_retained_count, - emqx_retained_max + emqx_retained_max, + emqx_delayed_count, + emqx_delayed_max ]. %% @doc Get stats by name. diff --git a/apps/emqx_opentelemetry/src/emqx_otel.erl b/apps/emqx_opentelemetry/src/emqx_otel.erl index 0b0e16cab..cad40ce82 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel.erl @@ -87,10 +87,17 @@ create_metric_views() -> create_gauge(Meter, VmGauge, fun ?MODULE:get_vm_gauge/1), ClusterGauge = [{'node.running', 0}, {'node.stopped', 0}], create_gauge(Meter, ClusterGauge, fun ?MODULE:get_cluster_gauge/1), - Metrics = lists:map(fun({K, V}) -> {K, V, unit(K)} end, emqx_metrics:all()), + Metrics = lists:map(fun({K, V}) -> {to_metric_name(K), V, unit(K)} end, emqx_metrics:all()), create_counter(Meter, Metrics, fun ?MODULE:get_metric_counter/1), ok. +to_metric_name('messages.dropped.await_pubrel_timeout') -> + 'messages.dropped.expired'; +to_metric_name('packets.connect.received') -> + 'packets.connect'; +to_metric_name(Name) -> + Name. + unit(K) -> case lists:member(K, bytes_metrics()) of true -> kb; diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 66ab9f61f..fc42d89ff 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -231,6 +231,10 @@ emqx_collect(emqx_sessions_count, Stats) -> gauge_metric(?C('sessions.count', Stats)); emqx_collect(emqx_sessions_max, Stats) -> gauge_metric(?C('sessions.max', Stats)); +emqx_collect(emqx_channels_count, Stats) -> + gauge_metric(?C('channels.count', Stats)); +emqx_collect(emqx_channels_max, Stats) -> + gauge_metric(?C('channels.max', Stats)); %% pub/sub stats emqx_collect(emqx_topics_count, Stats) -> gauge_metric(?C('topics.count', Stats)); @@ -257,6 +261,11 @@ emqx_collect(emqx_retained_count, Stats) -> gauge_metric(?C('retained.count', Stats)); emqx_collect(emqx_retained_max, Stats) -> gauge_metric(?C('retained.max', Stats)); +%% delayed +emqx_collect(emqx_delayed_count, Stats) -> + gauge_metric(?C('delayed.count', Stats)); +emqx_collect(emqx_delayed_max, Stats) -> + gauge_metric(?C('delayed.max', Stats)); %%-------------------------------------------------------------------- %% Metrics - packets & bytes @@ -411,7 +420,10 @@ emqx_collect(emqx_delivery_dropped_expired, Stats) -> counter_metric(?C('delivery.dropped.expired', Stats)); %%-------------------------------------------------------------------- %% Metrics - client - +emqx_collect(emqx_client_connect, Stats) -> + counter_metric(?C('client.connect', Stats)); +emqx_collect(emqx_client_connack, Stats) -> + counter_metric(?C('client.connack', Stats)); emqx_collect(emqx_client_connected, Stats) -> counter_metric(?C('client.connected', Stats)); emqx_collect(emqx_client_authenticate, Stats) -> @@ -460,6 +472,14 @@ emqx_collect(emqx_authorization_deny, Stats) -> counter_metric(?C('authorization.deny', Stats)); emqx_collect(emqx_authorization_cache_hit, Stats) -> counter_metric(?C('authorization.cache_hit', Stats)); +emqx_collect(emqx_authorization_superuser, Stats) -> + counter_metric(?C('authorization.superuser', Stats)); +emqx_collect(emqx_authorization_nomatch, Stats) -> + counter_metric(?C('authorization.nomatch', Stats)); +emqx_collect(emqx_authorization_matched_allow, Stats) -> + counter_metric(?C('authorization.matched_allow', Stats)); +emqx_collect(emqx_authorization_matched_deny, Stats) -> + counter_metric(?C('authorization.matched_deny', Stats)); %%-------------------------------------------------------------------- %% Metrics - authn emqx_collect(emqx_authentication_success, Stats) -> @@ -566,7 +586,11 @@ emqx_metrics_acl() -> [ emqx_authorization_allow, emqx_authorization_deny, - emqx_authorization_cache_hit + emqx_authorization_cache_hit, + emqx_authorization_superuser, + emqx_authorization_nomatch, + emqx_authorization_matched_allow, + emqx_authorization_matched_deny ]. emqx_metrics_authn() -> @@ -609,6 +633,8 @@ emqx_metrics_delivery() -> emqx_metrics_client() -> [ + emqx_client_connect, + emqx_client_connack, emqx_client_connected, emqx_client_authenticate, emqx_client_auth_anonymous, From 6bb15541c795984e205204ca2117d7fa772aeed3 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 23 Aug 2023 15:25:54 +0800 Subject: [PATCH 041/119] feat: rename olp metrics to overload_protection metrics --- apps/emqx/src/config/emqx_config_zones.erl | 11 +++++++++ apps/emqx/src/emqx_metrics.erl | 24 ++++++++++++-------- apps/emqx/src/emqx_olp.erl | 20 ++++++++-------- apps/emqx/src/emqx_quic_connection.erl | 2 +- apps/emqx_opentelemetry/src/emqx_otel.erl | 17 +++++++++++++- apps/emqx_prometheus/src/emqx_prometheus.erl | 22 +++++------------- 6 files changed, 58 insertions(+), 38 deletions(-) diff --git a/apps/emqx/src/config/emqx_config_zones.erl b/apps/emqx/src/config/emqx_config_zones.erl index 57e2824ff..bcc3c1d99 100644 --- a/apps/emqx/src/config/emqx_config_zones.erl +++ b/apps/emqx/src/config/emqx_config_zones.erl @@ -19,6 +19,7 @@ %% API -export([add_handler/0, remove_handler/0, pre_config_update/3]). +-export([is_olp_enabled/0]). -define(ZONES, [zones]). @@ -33,3 +34,13 @@ remove_handler() -> %% replace the old config with the new config pre_config_update(?ZONES, NewRaw, _OldRaw) -> {ok, NewRaw}. + +is_olp_enabled() -> + maps:fold( + fun + (_, #{overload_protection := #{enable := true}}, _Acc) -> true; + (_, _, Acc) -> Acc + end, + false, + emqx_conf:get([zones], #{}) + ). diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index 21a114c0f..b3400f371 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -67,6 +67,7 @@ terminate/2, code_change/3 ]). +-export([olp_metrics/0]). %% BACKW: v4.3.0 -export([upgrade_retained_delayed_counter_type/0]). @@ -269,13 +270,16 @@ %% Overload protetion counters -define(OLP_METRICS, [ - {counter, 'olp.delay.ok'}, - {counter, 'olp.delay.timeout'}, - {counter, 'olp.hbn'}, - {counter, 'olp.gc'}, - {counter, 'olp.new_conn'} + {counter, 'overload_protection.delay.ok'}, + {counter, 'overload_protection.delay.timeout'}, + {counter, 'overload_protection.hibernation'}, + {counter, 'overload_protection.gc'}, + {counter, 'overload_protection.new_conn'} ]). +olp_metrics() -> + lists:map(fun({_, Metric}) -> Metric end, ?OLP_METRICS). + -record(state, {next_idx = 1}). -record(metric, {name, type, idx}). @@ -701,9 +705,9 @@ reserved_idx('authorization.cache_hit') -> 302; reserved_idx('authentication.success') -> 310; reserved_idx('authentication.success.anonymous') -> 311; reserved_idx('authentication.failure') -> 312; -reserved_idx('olp.delay.ok') -> 400; -reserved_idx('olp.delay.timeout') -> 401; -reserved_idx('olp.hbn') -> 402; -reserved_idx('olp.gc') -> 403; -reserved_idx('olp.new_conn') -> 404; +reserved_idx('overload_protection.delay.ok') -> 400; +reserved_idx('overload_protection.delay.timeout') -> 401; +reserved_idx('overload_protection.hibernation') -> 402; +reserved_idx('overload_protection.gc') -> 403; +reserved_idx('overload_protection.new_conn') -> 404; reserved_idx(_) -> undefined. diff --git a/apps/emqx/src/emqx_olp.erl b/apps/emqx/src/emqx_olp.erl index 5a4775896..5ca35d8b4 100644 --- a/apps/emqx/src/emqx_olp.erl +++ b/apps/emqx/src/emqx_olp.erl @@ -38,11 +38,11 @@ | backoff_new_conn. -type cnt_name() :: - 'olp.delay.ok' - | 'olp.delay.timeout' - | 'olp.hbn' - | 'olp.gc' - | 'olp.new_conn'. + 'overload_protection.delay.ok' + | 'overload_protection.delay.timeout' + | 'overload_protection.hibernation' + | 'overload_protection.gc' + | 'overload_protection.new_conn'. -define(overload_protection, overload_protection). @@ -63,10 +63,10 @@ backoff(Zone) -> false -> false; ok -> - emqx_metrics:inc('olp.delay.ok'), + emqx_metrics:inc('overload_protection.delay.ok'), ok; timeout -> - emqx_metrics:inc('olp.delay.timeout'), + emqx_metrics:inc('overload_protection.delay.timeout'), timeout end; _ -> @@ -76,18 +76,18 @@ backoff(Zone) -> %% @doc If forceful GC should be skipped when the system is overloaded. -spec backoff_gc(Zone :: atom()) -> boolean(). backoff_gc(Zone) -> - do_check(Zone, ?FUNCTION_NAME, 'olp.gc'). + do_check(Zone, ?FUNCTION_NAME, 'overload_protection.gc'). %% @doc If hibernation should be skipped when the system is overloaded. -spec backoff_hibernation(Zone :: atom()) -> boolean(). backoff_hibernation(Zone) -> - do_check(Zone, ?FUNCTION_NAME, 'olp.hbn'). + do_check(Zone, ?FUNCTION_NAME, 'overload_protection.hibernation'). %% @doc Returns {error, overloaded} if new connection should be %% closed when system is overloaded. -spec backoff_new_conn(Zone :: atom()) -> ok | {error, overloaded}. backoff_new_conn(Zone) -> - case do_check(Zone, ?FUNCTION_NAME, 'olp.new_conn') of + case do_check(Zone, ?FUNCTION_NAME, 'overload_protection.new_conn') of true -> {error, overloaded}; false -> diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index a77ec28f2..7ddf05af3 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -118,7 +118,7 @@ new_conn( {stop, stream_accept_error, S} end; true -> - emqx_metrics:inc('olp.new_conn'), + emqx_metrics:inc('overload_protection.new_conn'), _ = quicer:async_shutdown_connection( Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, diff --git a/apps/emqx_opentelemetry/src/emqx_otel.erl b/apps/emqx_opentelemetry/src/emqx_otel.erl index cad40ce82..418c01c8d 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel.erl @@ -87,10 +87,25 @@ create_metric_views() -> create_gauge(Meter, VmGauge, fun ?MODULE:get_vm_gauge/1), ClusterGauge = [{'node.running', 0}, {'node.stopped', 0}], create_gauge(Meter, ClusterGauge, fun ?MODULE:get_cluster_gauge/1), - Metrics = lists:map(fun({K, V}) -> {to_metric_name(K), V, unit(K)} end, emqx_metrics:all()), + Metrics0 = filter_olp_metrics(emqx_metrics:all()), + Metrics = lists:map(fun({K, V}) -> {to_metric_name(K), V, unit(K)} end, Metrics0), create_counter(Meter, Metrics, fun ?MODULE:get_metric_counter/1), ok. +filter_olp_metrics(Metrics) -> + case emqx_config_zones:is_olp_enabled() of + true -> + Metrics; + false -> + OlpMetrics = emqx_metrics:olp_metrics(), + lists:filter( + fun({K, _}) -> + not lists:member(K, OlpMetrics) + end, + Metrics + ) + end. + to_metric_name('messages.dropped.await_pubrel_timeout') -> 'messages.dropped.expired'; to_metric_name('packets.connect.received') -> diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index fc42d89ff..e9030d3ed 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -455,15 +455,15 @@ emqx_collect(emqx_session_terminated, Stats) -> %% Metrics - overload protection emqx_collect(emqx_overload_protection_delay_ok, Stats) -> - counter_metric(?C('olp.delay.ok', Stats)); + counter_metric(?C('overload_protection.delay.ok', Stats)); emqx_collect(emqx_overload_protection_delay_timeout, Stats) -> - counter_metric(?C('olp.delay.timeout', Stats)); + counter_metric(?C('overload_protection.delay.timeout', Stats)); emqx_collect(emqx_overload_protection_hibernation, Stats) -> - counter_metric(?C('olp.hbn', Stats)); + counter_metric(?C('overload_protection.hibernation', Stats)); emqx_collect(emqx_overload_protection_gc, Stats) -> - counter_metric(?C('olp.gc', Stats)); + counter_metric(?C('overload_protection.gc', Stats)); emqx_collect(emqx_overload_protection_new_conn, Stats) -> - counter_metric(?C('olp.new_conn', Stats)); + counter_metric(?C('overload_protection.new_conn', Stats)); %%-------------------------------------------------------------------- %% Metrics - acl emqx_collect(emqx_authorization_allow, Stats) -> @@ -559,7 +559,7 @@ emqx_metrics_packets() -> ]. emqx_metrics_olp() -> - case is_olp_enabled() of + case emqx_config_zones:is_olp_enabled() of true -> [ emqx_overload_protection_delay_ok, @@ -572,16 +572,6 @@ emqx_metrics_olp() -> [] end. -is_olp_enabled() -> - maps:fold( - fun - (_, #{overload_protection := #{enable := true}}, _Acc) -> true; - (_, _, Acc) -> Acc - end, - false, - emqx_conf:get([zones], #{}) - ). - emqx_metrics_acl() -> [ emqx_authorization_allow, From 58be7f709b32d2489accdd80cfb66a019af4b5f5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 23 Aug 2023 16:45:41 +0800 Subject: [PATCH 042/119] fix: stop otel deps application in terminate crashed --- apps/emqx_opentelemetry/src/emqx_otel.erl | 4 +--- apps/emqx_opentelemetry/src/emqx_otel_sup.erl | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/emqx_opentelemetry/src/emqx_otel.erl b/apps/emqx_opentelemetry/src/emqx_otel.erl index 0b0e16cab..a9d9af0ea 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel.erl @@ -17,7 +17,7 @@ -module(emqx_otel). -include_lib("emqx/include/logger.hrl"). --export([start_link/1]). +-export([start_link/1, cleanup/0]). -export([get_cluster_gauge/1, get_stats_gauge/1, get_vm_gauge/1, get_metric_counter/1]). -export([init/1, handle_continue/2, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). @@ -25,7 +25,6 @@ start_link(Conf) -> gen_server:start_link({local, ?MODULE}, ?MODULE, Conf, []). init(Conf) -> - erlang:process_flag(trap_exit, true), {ok, #{}, {continue, {setup, Conf}}}. handle_continue({setup, Conf}, State) -> @@ -42,7 +41,6 @@ handle_info(_Msg, State) -> {noreply, State}. terminate(_Reason, _State) -> - cleanup(), ok. setup(Conf = #{enable := true}) -> diff --git a/apps/emqx_opentelemetry/src/emqx_otel_sup.erl b/apps/emqx_opentelemetry/src/emqx_otel_sup.erl index 2240cca03..6d04a0c2e 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_sup.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_sup.erl @@ -42,6 +42,7 @@ start_otel(Conf) -> -spec stop_otel() -> ok | {error, term()}. stop_otel() -> + ok = emqx_otel:cleanup(), case supervisor:terminate_child(?MODULE, ?WORKER) of ok -> supervisor:delete_child(?MODULE, ?WORKER); {error, not_found} -> ok; From d287b736ec77ba19b1b180f477330d9b1543aefb Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 23 Aug 2023 15:30:18 +0800 Subject: [PATCH 043/119] chore: add changelog for 11497 --- apps/emqx/src/config/emqx_config_zones.erl | 2 +- changes/ce/feat-11497.en.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changes/ce/feat-11497.en.md diff --git a/apps/emqx/src/config/emqx_config_zones.erl b/apps/emqx/src/config/emqx_config_zones.erl index bcc3c1d99..c367e2198 100644 --- a/apps/emqx/src/config/emqx_config_zones.erl +++ b/apps/emqx/src/config/emqx_config_zones.erl @@ -42,5 +42,5 @@ is_olp_enabled() -> (_, _, Acc) -> Acc end, false, - emqx_conf:get([zones], #{}) + emqx_config:get([zones], #{}) ). diff --git a/changes/ce/feat-11497.en.md b/changes/ce/feat-11497.en.md new file mode 100644 index 000000000..30ef73fb1 --- /dev/null +++ b/changes/ce/feat-11497.en.md @@ -0,0 +1,2 @@ +Enhanced broker metrics collection and export by adding new metrics for messages, overload protection, authorization, authentication, +and improving naming consistency for OpenTelemetry. From 92a293a67b5ed8d4e40ceb74668babee7f9a86a9 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 23 Aug 2023 17:01:50 +0300 Subject: [PATCH 044/119] chore(ci): pin pytest-retry version --- .ci/docker-compose-file/python/pytest.sh | 2 +- .github/workflows/run_helm_tests.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/docker-compose-file/python/pytest.sh b/.ci/docker-compose-file/python/pytest.sh index 924c30212..649357cff 100755 --- a/.ci/docker-compose-file/python/pytest.sh +++ b/.ci/docker-compose-file/python/pytest.sh @@ -20,7 +20,7 @@ fi apk update && apk add git curl git clone -b develop-5.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing -pip install pytest==7.1.2 pytest-retry +pip install pytest==7.1.2 pytest-retry==1.3.0 pytest --retries 3 -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$TARGET_HOST" RESULT=$? diff --git a/.github/workflows/run_helm_tests.yaml b/.github/workflows/run_helm_tests.yaml index 1106b6057..ba3ebee5a 100644 --- a/.github/workflows/run_helm_tests.yaml +++ b/.github/workflows/run_helm_tests.yaml @@ -121,7 +121,7 @@ jobs: path: paho.mqtt.testing - name: install pytest run: | - pip install pytest==7.1.2 pytest-retry + pip install pytest==7.1.2 pytest-retry==1.3.0 echo "$HOME/.local/bin" >> $GITHUB_PATH - name: run paho test timeout-minutes: 10 From 84dd3b83bbc99c81088b6981ce56e2a34802ef4d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 17 Aug 2023 18:34:56 +0800 Subject: [PATCH 045/119] fix: hocon converter not working when make_serializable is true(check_plain) --- apps/emqx/src/emqx_schema.erl | 28 ++--- .../emqx/test/emqx_listeners_update_SUITE.erl | 116 ++++++++++++++++++ .../src/emqx_bridge_gcp_pubsub.erl | 10 +- .../src/emqx_prometheus_schema.erl | 12 +- .../test/emqx_prometheus_api_SUITE.erl | 32 ++++- 5 files changed, 172 insertions(+), 26 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 07e909ae0..ff1446b6d 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2314,18 +2314,7 @@ ciphers_schema(Default) -> hoconsc:array(string()), #{ default => default_ciphers(Default), - converter => fun - (undefined) -> - []; - (<<>>) -> - []; - ("") -> - []; - (Ciphers) when is_binary(Ciphers) -> - binary:split(Ciphers, <<",">>, [global]); - (Ciphers) when is_list(Ciphers) -> - Ciphers - end, + converter => fun converter_ciphers/2, validator => case Default =:= quic of %% quic has openssl statically linked @@ -2336,6 +2325,14 @@ ciphers_schema(Default) -> } ). +converter_ciphers(undefined, _Opts) -> + []; +converter_ciphers(<<>>, _Opts) -> + []; +converter_ciphers(Ciphers, _Opts) when is_list(Ciphers) -> Ciphers; +converter_ciphers(Ciphers, _Opts) when is_binary(Ciphers) -> + binary:split(Ciphers, <<",">>, [global]). + default_ciphers(Which) -> lists:map( fun erlang:iolist_to_binary/1, @@ -3125,9 +3122,10 @@ quic_feature_toggle(Desc) -> importance => ?IMPORTANCE_HIDDEN, required => false, converter => fun - (true) -> 1; - (false) -> 0; - (Other) -> Other + (Val, #{make_serializable := true}) -> Val; + (true, _Opts) -> 1; + (false, _Opts) -> 0; + (Other, _Opts) -> Other end } ). diff --git a/apps/emqx/test/emqx_listeners_update_SUITE.erl b/apps/emqx/test/emqx_listeners_update_SUITE.erl index c16a26f3a..5c51d990a 100644 --- a/apps/emqx/test/emqx_listeners_update_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_update_SUITE.erl @@ -116,6 +116,122 @@ t_update_conf(_Conf) -> ?assert(is_running('wss:default')), ok. +t_update_empty_ssl_options_conf(_Conf) -> + Raw = emqx:get_raw_config(?LISTENERS), + Raw1 = emqx_utils_maps:deep_put( + [<<"tcp">>, <<"default">>, <<"bind">>], Raw, <<"127.0.0.1:1883">> + ), + Raw2 = emqx_utils_maps:deep_put( + [<<"ssl">>, <<"default">>, <<"bind">>], Raw1, <<"127.0.0.1:8883">> + ), + Raw3 = emqx_utils_maps:deep_put( + [<<"ws">>, <<"default">>, <<"bind">>], Raw2, <<"0.0.0.0:8083">> + ), + Raw4 = emqx_utils_maps:deep_put( + [<<"wss">>, <<"default">>, <<"bind">>], Raw3, <<"127.0.0.1:8084">> + ), + Raw5 = emqx_utils_maps:deep_put( + [<<"ssl">>, <<"default">>, <<"ssl_options">>, <<"cacertfile">>], Raw4, <<"">> + ), + Raw6 = emqx_utils_maps:deep_put( + [<<"wss">>, <<"default">>, <<"ssl_options">>, <<"cacertfile">>], Raw5, <<"">> + ), + Raw7 = emqx_utils_maps:deep_put( + [<<"wss">>, <<"default">>, <<"ssl_options">>, <<"ciphers">>], Raw6, <<"">> + ), + Raw8 = emqx_utils_maps:deep_put( + [<<"ssl">>, <<"default">>, <<"ssl_options">>, <<"ciphers">>], + Raw7, + <<"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256">> + ), + ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw8)), + ?assertMatch( + #{ + <<"tcp">> := #{<<"default">> := #{<<"bind">> := <<"127.0.0.1:1883">>}}, + <<"ssl">> := #{ + <<"default">> := #{ + <<"bind">> := <<"127.0.0.1:8883">>, + <<"ssl_options">> := #{ + <<"cacertfile">> := <<"">>, + <<"ciphers">> := <<"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256">> + } + } + }, + <<"ws">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:8083">>}}, + <<"wss">> := #{ + <<"default">> := #{ + <<"bind">> := <<"127.0.0.1:8084">>, + <<"ssl_options">> := #{ + <<"cacertfile">> := <<"">>, + <<"ciphers">> := <<"">> + } + } + } + }, + emqx:get_raw_config(?LISTENERS) + ), + BindTcp = {{127, 0, 0, 1}, 1883}, + BindSsl = {{127, 0, 0, 1}, 8883}, + BindWs = {{0, 0, 0, 0}, 8083}, + BindWss = {{127, 0, 0, 1}, 8084}, + ?assertMatch( + #{ + tcp := #{default := #{bind := BindTcp}}, + ssl := #{ + default := #{ + bind := BindSsl, + ssl_options := #{ + cacertfile := <<"">>, + ciphers := ["TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256"] + } + } + }, + ws := #{default := #{bind := BindWs}}, + wss := #{ + default := #{ + bind := BindWss, + ssl_options := #{ + cacertfile := <<"">>, + ciphers := [] + } + } + } + }, + emqx:get_config(?LISTENERS) + ), + ?assertError(not_found, current_conns(<<"tcp:default">>, {{0, 0, 0, 0}, 1883})), + ?assertError(not_found, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})), + + ?assertEqual(0, current_conns(<<"tcp:default">>, BindTcp)), + ?assertEqual(0, current_conns(<<"ssl:default">>, BindSsl)), + + ?assertEqual({0, 0, 0, 0}, proplists:get_value(ip, ranch:info('ws:default'))), + ?assertEqual({127, 0, 0, 1}, proplists:get_value(ip, ranch:info('wss:default'))), + ?assert(is_running('ws:default')), + ?assert(is_running('wss:default')), + + Raw9 = emqx_utils_maps:deep_put( + [<<"ssl">>, <<"default">>, <<"ssl_options">>, <<"ciphers">>], Raw7, [ + "TLS_AES_256_GCM_SHA384", + "TLS_AES_128_GCM_SHA256", + "TLS_CHACHA20_POLY1305_SHA256" + ] + ), + ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw9)), + + BadRaw = emqx_utils_maps:deep_put( + [<<"ssl">>, <<"default">>, <<"ssl_options">>, <<"keyfile">>], Raw4, <<"">> + ), + ?assertMatch( + {error, + {bad_ssl_config, #{ + reason := pem_file_path_or_string_is_required, + which_options := [[<<"keyfile">>]] + }}}, + emqx:update_config(?LISTENERS, BadRaw) + ), + ok. + t_add_delete_conf(_Conf) -> Raw = emqx:get_raw_config(?LISTENERS), %% add diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl index 7324ff723..685fd3397 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl @@ -18,7 +18,7 @@ ]). -export([ service_account_json_validator/1, - service_account_json_converter/1 + service_account_json_converter/2 ]). %% emqx_bridge_enterprise "unofficial" API @@ -105,7 +105,7 @@ fields(connector_config) -> #{ required => true, validator => fun ?MODULE:service_account_json_validator/1, - converter => fun ?MODULE:service_account_json_converter/1, + converter => fun ?MODULE:service_account_json_converter/2, sensitive => true, desc => ?DESC("service_account_json") } @@ -398,7 +398,9 @@ service_account_json_validator(Map) -> {error, #{missing_keys => MissingKeys}} end. -service_account_json_converter(Map) when is_map(Map) -> +service_account_json_converter(Val, #{make_serializable := true}) -> + Val; +service_account_json_converter(Map, _Opts) when is_map(Map) -> ExpectedKeys = [ <<"type">>, <<"project_id">>, @@ -407,7 +409,7 @@ service_account_json_converter(Map) when is_map(Map) -> <<"client_email">> ], maps:with(ExpectedKeys, Map); -service_account_json_converter(Val) -> +service_account_json_converter(Val, _Opts) -> Val. consumer_topic_mapping_validator(_TopicMapping = []) -> diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 41d74aa20..f34675c0b 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -26,7 +26,7 @@ fields/1, desc/1, translation/1, - convert_headers/1, + convert_headers/2, validate_push_gateway_server/1 ]). @@ -61,7 +61,7 @@ fields("prometheus") -> #{ default => #{}, required => false, - converter => fun ?MODULE:convert_headers/1, + converter => fun ?MODULE:convert_headers/2, desc => ?DESC(headers) } )}, @@ -155,9 +155,11 @@ fields("prometheus") -> desc("prometheus") -> ?DESC(prometheus); desc(_) -> undefined. -convert_headers(<<>>) -> +convert_headers(Headers, #{make_serializable := true}) -> + Headers; +convert_headers(<<>>, _Opts) -> []; -convert_headers(Headers) when is_map(Headers) -> +convert_headers(Headers, _Opts) when is_map(Headers) -> maps:fold( fun(K, V, Acc) -> [{binary_to_list(K), binary_to_list(V)} | Acc] @@ -165,7 +167,7 @@ convert_headers(Headers) when is_map(Headers) -> [], Headers ); -convert_headers(Headers) when is_list(Headers) -> +convert_headers(Headers, _Opts) when is_list(Headers) -> Headers. validate_push_gateway_server(Url) -> diff --git a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl index c7d336554..eb909baf5 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl @@ -75,13 +75,22 @@ t_prometheus_api(_) -> <<"vm_statistics_collector">> := _, <<"vm_system_info_collector">> := _, <<"vm_memory_collector">> := _, - <<"vm_msacc_collector">> := _ + <<"vm_msacc_collector">> := _, + <<"headers">> := _ }, Conf ), #{<<"enable">> := Enable} = Conf, ?assertEqual(Enable, undefined =/= erlang:whereis(emqx_prometheus)), - NewConf = Conf#{<<"interval">> => <<"2s">>, <<"vm_statistics_collector">> => <<"enabled">>}, + + NewConf = Conf#{ + <<"interval">> => <<"2s">>, + <<"vm_statistics_collector">> => <<"enabled">>, + <<"headers">> => #{ + <<"test-str1">> => <<"test-value">>, + <<"test-str2">> => <<"42">> + } + }, {ok, Response2} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf), Conf2 = emqx_utils_json:decode(Response2, [return_maps]), @@ -106,6 +115,25 @@ t_prometheus_api(_) -> ] ), + ?assertMatch( + #{ + <<"headers">> := #{ + <<"test-str1">> := <<"test-value">>, + <<"test-str2">> := <<"42">> + } + }, + emqx_config:get_raw([prometheus]) + ), + ?assertMatch( + #{ + headers := [ + {"test-str2", "42"}, + {"test-str1", "test-value"} + ] + }, + emqx_config:get([prometheus]) + ), + NewConf1 = Conf#{<<"enable">> => (not Enable)}, {ok, _Response3} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf1), ?assertEqual((not Enable), undefined =/= erlang:whereis(emqx_prometheus)), From 8d2be27adee4cb23fba82b4f34f24048d476939a Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 17 Aug 2023 22:56:06 +0800 Subject: [PATCH 046/119] chore: add changlog for 11466 --- apps/emqx/src/emqx_metrics_worker.erl | 2 +- changes/ce/fix-11466.en.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/ce/fix-11466.en.md diff --git a/apps/emqx/src/emqx_metrics_worker.erl b/apps/emqx/src/emqx_metrics_worker.erl index 5f41346cb..a57a2f2f0 100644 --- a/apps/emqx/src/emqx_metrics_worker.erl +++ b/apps/emqx/src/emqx_metrics_worker.erl @@ -495,7 +495,7 @@ terminate(_Reason, #state{metric_ids = MIDs}) -> stop(Name) -> try - gen_server:stop(Name) + gen_server:stop(Name, normal, 10000) catch exit:noproc -> ok; diff --git a/changes/ce/fix-11466.en.md b/changes/ce/fix-11466.en.md new file mode 100644 index 000000000..80679d444 --- /dev/null +++ b/changes/ce/fix-11466.en.md @@ -0,0 +1 @@ +Fixed a crash that occurred when setting the `ssl_options.ciphers` configuration option to an empty string (""). From 6085442958dc5bbeaff9918e6b5e798277bc6bd8 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 24 Aug 2023 15:35:21 +0800 Subject: [PATCH 047/119] fix(ldap): expose request_timeout to prevent infinite hang --- apps/emqx_ldap/src/emqx_ldap.erl | 16 +++++++++++++--- rel/i18n/emqx_ldap.hocon | 6 ++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index 73a0a363c..d14ddf97d 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -70,7 +70,12 @@ fields(config) -> example => <<"(& (objectClass=mqttUser) (uid=${username}))">>, validator => fun emqx_schema:non_empty_string/1 } - )} + )}, + {request_timeout, + ?HOCON(emqx_schema:timeout_duration_ms(), #{ + desc => ?DESC(request_timeout), + default => <<"5s">> + })} ] ++ emqx_connector_schema_lib:ssl_fields(). server() -> @@ -161,10 +166,15 @@ do_get_status(Conn) -> %% =================================================================== connect(Options) -> - #{hostname := Host, username := Username, password := Password} = + #{ + hostname := Host, + username := Username, + password := Password, + request_timeout := RequestTimeout + } = Conf = proplists:get_value(options, Options), OpenOpts = maps:to_list(maps:with([port, sslopts], Conf)), - case eldap:open([Host], [{log, fun log/3} | OpenOpts]) of + case eldap:open([Host], [{log, fun log/3}, {timeout, RequestTimeout} | OpenOpts]) of {ok, Handle} = Ret -> case eldap:simple_bind(Handle, Username, Password) of ok -> Ret; diff --git a/rel/i18n/emqx_ldap.hocon b/rel/i18n/emqx_ldap.hocon index 99e00e63a..204431907 100644 --- a/rel/i18n/emqx_ldap.hocon +++ b/rel/i18n/emqx_ldap.hocon @@ -23,4 +23,10 @@ The syntax of the filter follows RFC 4515 and also supports placeholders.""" filter.label: """Filter""" +request_timeout.desc: +"""Sets the maximum time in milliseconds that is used for each individual request.""" + +request_timeout.label: +"""Request Timeout""" + } From ae094e363caf2a02edd586b0168a1006c81d7289 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 15 Aug 2023 16:27:19 +0200 Subject: [PATCH 048/119] chore: fix a typo in log message --- apps/emqx_conf/src/emqx_cluster_rpc.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index bb154f8b5..934d7ef7a 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -649,7 +649,7 @@ do_wait_for_emqx_ready(N) -> ok -> ok; timeout -> - ?SLOG(warning, #{msg => "stil_waiting_for_emqx_app_to_be_ready"}), + ?SLOG(warning, #{msg => "still_waiting_for_emqx_app_to_be_ready"}), do_wait_for_emqx_ready(N - 1) end. From 6b152b3cb70af21558be7bd8be050d73d33c51af Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 20 Aug 2023 20:21:45 +0200 Subject: [PATCH 049/119] test: add more debug output --- apps/emqx/test/emqx_topic_index_SUITE.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/emqx/test/emqx_topic_index_SUITE.erl b/apps/emqx/test/emqx_topic_index_SUITE.erl index ade98acec..9058f3597 100644 --- a/apps/emqx/test/emqx_topic_index_SUITE.erl +++ b/apps/emqx/test/emqx_topic_index_SUITE.erl @@ -252,8 +252,9 @@ topic_matches_prop() -> ct:pal( "Topic name: ~p~n" "Index results: ~p~n" - "Topic match results:: ~p~n", - [Topic, Ids1, Ids2] + "Topic match results: ~p~n" + "Filters: ~p~n", + [Topic, Ids1, Ids2, Filters] ), false end From f4c8c6be5573804203dcd5d7e66ede487cfd19bf Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 17 Aug 2023 10:13:22 +0200 Subject: [PATCH 050/119] refactor(topic_index): optimize trie-search performance --- apps/emqx/src/emqx_topic_gbt.erl | 116 ++++++++ apps/emqx/src/emqx_topic_index.erl | 191 ++---------- apps/emqx/src/emqx_trie_search.erl | 344 ++++++++++++++++++++++ apps/emqx/test/emqx_topic_index_SUITE.erl | 190 +++++++----- 4 files changed, 593 insertions(+), 248 deletions(-) create mode 100644 apps/emqx/src/emqx_topic_gbt.erl create mode 100644 apps/emqx/src/emqx_trie_search.erl diff --git a/apps/emqx/src/emqx_topic_gbt.erl b/apps/emqx/src/emqx_topic_gbt.erl new file mode 100644 index 000000000..da6320bc7 --- /dev/null +++ b/apps/emqx/src/emqx_topic_gbt.erl @@ -0,0 +1,116 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc Topic index implemetation with gb_trees stored in persistent_term. +%% This is only suitable for a static set of topic or topic-filters. + +-module(emqx_topic_gbt). + +-export([new/0, new/1]). +-export([insert/4]). +-export([delete/3]). +-export([match/2]). +-export([matches/3]). + +-export([get_id/1]). +-export([get_topic/1]). +-export([get_record/2]). + +-type word() :: binary() | '+' | '#'. +-type key(ID) :: {[word()], {ID}}. +-type match(ID) :: key(ID). +-type name() :: any(). + +%% @private Only for testing. +-spec new() -> name(). +new() -> + new(test). + +%% @doc Create a new gb_tree and store it in the persitent_term with the +%% given name. +-spec new(name()) -> name(). +new(Name) -> + T = gb_trees:from_orddict([]), + true = gbt_update(Name, T), + Name. + +%% @doc Insert a new entry into the index that associates given topic filter to given +%% record ID, and attaches arbitrary record to the entry. This allows users to choose +%% between regular and "materialized" indexes, for example. +-spec insert(emqx_types:topic(), _ID, _Record, name()) -> true. +insert(Filter, ID, Record, Name) -> + Tree = gbt(Name), + Key = key(Filter, ID), + NewTree = gb_trees:enter(Key, Record, Tree), + true = gbt_update(Name, NewTree). + +%% @doc Delete an entry from the index that associates given topic filter to given +%% record ID. Deleting non-existing entry is not an error. +-spec delete(emqx_types:topic(), _ID, name()) -> true. +delete(Filter, ID, Name) -> + Tree = gbt(Name), + Key = key(Filter, ID), + NewTree = gb_trees:delete_any(Key, Tree), + true = gbt_update(Name, NewTree). + +%% @doc Match given topic against the index and return the first match, or `false` if +%% no match is found. +-spec match(emqx_types:topic(), name()) -> match(_ID) | false. +match(Topic, Name) -> + emqx_trie_search:match(Topic, make_nextf(Name)). + +%% @doc Match given topic against the index and return _all_ matches. +%% If `unique` option is given, return only unique matches by record ID. +matches(Topic, Name, Opts) -> + emqx_trie_search:matches(Topic, make_nextf(Name), Opts). + +%% @doc Extract record ID from the match. +-spec get_id(match(ID)) -> ID. +get_id(Key) -> + emqx_trie_search:get_id(Key). + +%% @doc Extract topic (or topic filter) from the match. +-spec get_topic(match(_ID)) -> emqx_types:topic(). +get_topic(Key) -> + emqx_trie_search:get_topic(Key). + +%% @doc Fetch the record associated with the match. +-spec get_record(match(_ID), name()) -> _Record. +get_record(Key, Name) -> + Gbt = gbt(Name), + gb_trees:get(Key, Gbt). + +key(TopicOrFilter, ID) -> + emqx_trie_search:make_key(TopicOrFilter, ID). + +gbt(Name) -> + persistent_term:get({?MODULE, Name}). + +gbt_update(Name, Tree) -> + persistent_term:put({?MODULE, Name}, Tree), + true. + +gbt_next(nil, _Input) -> + emqx_trie_search:ceiling(); +gbt_next({P, _V, _Smaller, Bigger}, K) when K >= P -> + gbt_next(Bigger, K); +gbt_next({P, _V, Smaller, _Bigger}, K) -> + NextKey = gbt_next(Smaller, K), + min(P, NextKey). + +make_nextf(Name) -> + {_SizeWeDontCare, TheTree} = gbt(Name), + fun(Key) -> gbt_next(TheTree, Key) end. diff --git a/apps/emqx/src/emqx_topic_index.erl b/apps/emqx/src/emqx_topic_index.erl index a6f662f74..e1da5b3b7 100644 --- a/apps/emqx/src/emqx_topic_index.erl +++ b/apps/emqx/src/emqx_topic_index.erl @@ -14,18 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% @doc Topic index for matching topics to topic filters. -%% -%% Works on top of ETS ordered_set table. Keys are tuples constructed from -%% parsed topic filters and record IDs, wrapped in a tuple to order them -%% strictly greater than unit tuple (`{}`). Existing table may be used if -%% existing keys will not collide with index keys. -%% -%% Designed to effectively answer questions like: -%% 1. Does any topic filter match given topic? -%% 2. Which records are associated with topic filters matching given topic? -%% 3. Which topic filters match given topic? -%% 4. Which record IDs are associated with topic filters matching given topic? +%% @doc Topic index implemetation with ETS table as ordered-set storage. -module(emqx_topic_index). @@ -39,186 +28,51 @@ -export([get_topic/1]). -export([get_record/2]). --type word() :: binary() | '+' | '#'. --type key(ID) :: {[word()], {ID}}. +-type key(ID) :: emqx_trie_search:key(ID). -type match(ID) :: key(ID). %% @doc Create a new ETS table suitable for topic index. %% Usable mostly for testing purposes. -spec new() -> ets:table(). new() -> - ets:new(?MODULE, [public, ordered_set, {read_concurrency, true}]). + T = ets:new(?MODULE, [public, ordered_set, {read_concurrency, true}]), + ets:insert(T, {emqx_trie_search:ceiling(), []}), + T. %% @doc Insert a new entry into the index that associates given topic filter to given %% record ID, and attaches arbitrary record to the entry. This allows users to choose %% between regular and "materialized" indexes, for example. -spec insert(emqx_types:topic(), _ID, _Record, ets:table()) -> true. insert(Filter, ID, Record, Tab) -> - ets:insert(Tab, {{words(Filter), {ID}}, Record}). + Key = key(Filter, ID), + true = ets:insert(Tab, {Key, Record}). %% @doc Delete an entry from the index that associates given topic filter to given %% record ID. Deleting non-existing entry is not an error. -spec delete(emqx_types:topic(), _ID, ets:table()) -> true. delete(Filter, ID, Tab) -> - ets:delete(Tab, {words(Filter), {ID}}). + true = ets:delete(Tab, key(Filter, ID)). %% @doc Match given topic against the index and return the first match, or `false` if %% no match is found. -spec match(emqx_types:topic(), ets:table()) -> match(_ID) | false. match(Topic, Tab) -> - {Words, RPrefix} = match_init(Topic), - match(Words, RPrefix, Tab). - -match(Words, RPrefix, Tab) -> - Prefix = lists:reverse(RPrefix), - match(ets:next(Tab, {Prefix, {}}), Prefix, Words, RPrefix, Tab). - -match(K, Prefix, Words, RPrefix, Tab) -> - case match_next(Prefix, K, Words) of - true -> - K; - skip -> - match(ets:next(Tab, K), Prefix, Words, RPrefix, Tab); - stop -> - false; - Matched -> - match_rest(Matched, Words, RPrefix, Tab) - end. - -match_rest([W1 | [W2 | _] = SLast], [W1 | [W2 | _] = Rest], RPrefix, Tab) -> - % NOTE - % Fast-forward through identical words in the topic and the last key suffixes. - % This should save us a few redundant `ets:next` calls at the cost of slightly - % more complex match patterns. - match_rest(SLast, Rest, [W1 | RPrefix], Tab); -match_rest(SLast, [W | Rest], RPrefix, Tab) when is_list(SLast) -> - match(Rest, [W | RPrefix], Tab); -match_rest(plus, [W | Rest], RPrefix, Tab) -> - % NOTE - % There's '+' in the key suffix, meaning we should consider 2 alternatives: - % 1. Match the rest of the topic as if there was '+' in the current position. - % 2. Skip this key and try to match the topic as it is. - case match(Rest, ['+' | RPrefix], Tab) of - Match = {_, _} -> - Match; - false -> - match(Rest, [W | RPrefix], Tab) - end; -match_rest(_, [], _RPrefix, _Tab) -> - false. + emqx_trie_search:match(Topic, make_nextf(Tab)). %% @doc Match given topic against the index and return _all_ matches. %% If `unique` option is given, return only unique matches by record ID. --spec matches(emqx_types:topic(), ets:table(), _Opts :: [unique]) -> [match(_ID)]. matches(Topic, Tab, Opts) -> - {Words, RPrefix} = match_init(Topic), - AccIn = - case Opts of - [unique | _] -> #{}; - [] -> [] - end, - Matches = matches(Words, RPrefix, AccIn, Tab), - case Matches of - #{} -> maps:values(Matches); - _ -> Matches - end. - -matches(Words, RPrefix, Acc, Tab) -> - Prefix = lists:reverse(RPrefix), - matches(ets:next(Tab, {Prefix, {}}), Prefix, Words, RPrefix, Acc, Tab). - -matches(Words, RPrefix, K = {Filter, _}, Acc, Tab) -> - Prefix = lists:reverse(RPrefix), - case Prefix > Filter of - true -> - % NOTE: Prefix already greater than the last key seen, need to `ets:next/2`. - matches(ets:next(Tab, {Prefix, {}}), Prefix, Words, RPrefix, Acc, Tab); - false -> - % NOTE: Prefix is still less than or equal to the last key seen, reuse it. - matches(K, Prefix, Words, RPrefix, Acc, Tab) - end. - -matches(K, Prefix, Words, RPrefix, Acc, Tab) -> - case match_next(Prefix, K, Words) of - true -> - matches(ets:next(Tab, K), Prefix, Words, RPrefix, match_add(K, Acc), Tab); - skip -> - matches(ets:next(Tab, K), Prefix, Words, RPrefix, Acc, Tab); - stop -> - Acc; - Matched -> - % NOTE: Prserve next key on the stack to save on `ets:next/2` calls. - matches_rest(Matched, Words, RPrefix, K, Acc, Tab) - end. - -matches_rest([W1 | [W2 | _] = SLast], [W1 | [W2 | _] = Rest], RPrefix, K, Acc, Tab) -> - % NOTE - % Fast-forward through identical words in the topic and the last key suffixes. - % This should save us a few redundant `ets:next` calls at the cost of slightly - % more complex match patterns. - matches_rest(SLast, Rest, [W1 | RPrefix], K, Acc, Tab); -matches_rest(SLast, [W | Rest], RPrefix, K, Acc, Tab) when is_list(SLast) -> - matches(Rest, [W | RPrefix], K, Acc, Tab); -matches_rest(plus, [W | Rest], RPrefix, K, Acc, Tab) -> - % NOTE - % There's '+' in the key suffix, meaning we should accumulate all matches from - % each of 2 branches: - % 1. Match the rest of the topic as if there was '+' in the current position. - % 2. Skip this key and try to match the topic as it is. - NAcc = matches(Rest, ['+' | RPrefix], K, Acc, Tab), - matches(Rest, [W | RPrefix], K, NAcc, Tab); -matches_rest(_, [], _RPrefix, _K, Acc, _Tab) -> - Acc. - -match_add(K = {_Filter, ID}, Acc = #{}) -> - % NOTE: ensuring uniqueness by record ID - Acc#{ID => K}; -match_add(K, Acc) -> - [K | Acc]. - -match_next(Prefix, {Filter, _ID}, Suffix) -> - match_filter(Prefix, Filter, Suffix); -match_next(_, '$end_of_table', _) -> - stop. - -match_filter([], [], []) -> - % NOTE: we matched the topic exactly - true; -match_filter([], [], _Suffix) -> - % NOTE: we matched the prefix, but there may be more matches next - skip; -match_filter([], ['#'], _Suffix) -> - % NOTE: naturally, '#' < '+', so this is already optimal for `match/2` - true; -match_filter([], ['+' | _], _Suffix) -> - plus; -match_filter([], [_H | _] = Rest, _Suffix) -> - Rest; -match_filter([H | T1], [H | T2], Suffix) -> - match_filter(T1, T2, Suffix); -match_filter([H1 | _], [H2 | _], _Suffix) when H2 > H1 -> - % NOTE: we're strictly past the prefix, no need to continue - stop. - -match_init(Topic) -> - case words(Topic) of - [W = <<"$", _/bytes>> | Rest] -> - % NOTE - % This will effectively skip attempts to match special topics to `#` or `+/...`. - {Rest, [W]}; - Words -> - {Words, []} - end. + emqx_trie_search:matches(Topic, make_nextf(Tab), Opts). %% @doc Extract record ID from the match. -spec get_id(match(ID)) -> ID. -get_id({_Filter, {ID}}) -> - ID. +get_id(Key) -> + emqx_trie_search:get_id(Key). %% @doc Extract topic (or topic filter) from the match. -spec get_topic(match(_ID)) -> emqx_types:topic(). -get_topic({Filter, _ID}) -> - emqx_topic:join(Filter). +get_topic(Key) -> + emqx_trie_search:get_topic(Key). %% @doc Fetch the record associated with the match. %% NOTE: Only really useful for ETS tables where the record ID is the first element. @@ -226,17 +80,8 @@ get_topic({Filter, _ID}) -> get_record(K, Tab) -> ets:lookup_element(Tab, K, 2). -%% +key(TopicOrFilter, ID) -> + emqx_trie_search:make_key(TopicOrFilter, ID). --spec words(emqx_types:topic()) -> [word()]. -words(Topic) when is_binary(Topic) -> - % NOTE - % This is almost identical to `emqx_topic:words/1`, but it doesn't convert empty - % tokens to ''. This is needed to keep ordering of words consistent with what - % `match_filter/3` expects. - [word(W) || W <- emqx_topic:tokens(Topic)]. - --spec word(binary()) -> word(). -word(<<"+">>) -> '+'; -word(<<"#">>) -> '#'; -word(Bin) -> Bin. +make_nextf(Tab) -> + fun(Key) -> ets:next(Tab, Key) end. diff --git a/apps/emqx/src/emqx_trie_search.erl b/apps/emqx/src/emqx_trie_search.erl new file mode 100644 index 000000000..8d9463c26 --- /dev/null +++ b/apps/emqx/src/emqx_trie_search.erl @@ -0,0 +1,344 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc Topic index for matching topics to topic filters. +%% +%% Works on top of ETS ordered_set table. Keys are tuples constructed from +%% parsed topic filters and record IDs, wrapped in a tuple to order them +%% strictly greater than unit tuple (`{}`). Existing table may be used if +%% existing keys will not collide with index keys. +%% +%% Designed to effectively answer questions like: +%% 1. Does any topic filter match given topic? +%% 2. Which records are associated with topic filters matching given topic? +%% 3. Which topic filters match given topic? +%% 4. Which record IDs are associated with topic filters matching given topic? +%% +%% Trie-search algorithm: +%% +%% Given a 3-level topic (e.g. a/b/c), if we leave out '#' for now, +%% all possible subscriptions of a/b/c can be enumerated as below: +%% +%% a/b/c +%% a/b/+ +%% a/+/c <--- subscribed +%% a/+/+ +%% +/b/c <--- subscribed +%% +/b/+ +%% +/+/c +%% +/+/+ <--- start searching upward from here +%% +%% Let's name this search space "Space1". +%% If we brute-force it, the scope would be 8 (2^3). +%% Meaning this has O(2^N) complexity (N being the level of topics). +%% +%% This clearly isn't going to work. +%% Should we then try to enumerate all subscribers instead? +%% If there are also other subscriptions, e.g. "+/x/y" and "+/b/0" +%% +%% a/+/c <--- match of a/b/c +%% +/x/n +%% ... +%% +/x/2 +%% +/x/1 +%% +/b/c <--- match of a/b/c +%% +/b/1 +%% +/b/0 +%% +%% Let's name it "Space2". +%% +%% This has O(M * L) complexity (M being the total number of subscriptions, +%% and L being the number of topic levels). +%% This is usually a lot smaller than "Space1", but still not very effective +%% if the collection size is e.g. 1 million. +%% +%% To make it more effective, we'll need to combine the two algorithms: +%% Use the ordered subscription topics' prefixes as starting points to make +%% guesses about whether or not the next word can be a '+', and skip-over +%% to the next possible match. +%% +%% NOTE: A prerequisite of the ordered collection is, it should be able +%% to find the *immediate-next* topic/filter with a given prefix. +%% +%% In the above example, we start from "+/b/0". When comparing "+/b/0" +%% with "a/b/c", we know the matching prefix is "+/b", meaning we can +%% start guessing if the next word is '+' or 'c': +%% * It can't be '+' because '+' < '0' +%% * It might be 'c' because 'c' > '0' +%% +%% So, we try to jump to the next topic which has a prefix of "+/b/c" +%% (this effectively means skipping over "+/b/1"). +%% +%% After "+/b/c" is found to be a matching filter, we move up: +%% * The next possible match is "a/+/+" according to Space1 +%% * The next subscription is "+/x/1" according to Space2 +%% +%% "a/+/+" is lexicographically greater than "+/x/+", so let's jump to +%% the immediate-next of 'a/+/+', which is "a/+/c", allowing us to skip +%% over all the ones starting with "+/x". +%% +%% If we take '#' into consideration, it's only one extra comparison to see +%% if a filter ends with '#'. +%% +%% In summary, the complexity of this algorithm is O(N * L) +%% N being the number of total matches, and L being the level of the topic. + +-module(emqx_trie_search). + +-export([ceiling/0, make_key/2]). +-export([match/2, matches/3, get_id/1, get_topic/1]). +-export_type([key/1, word/0, nextf/0, opts/0]). + +-type word() :: binary() | '+' | '#'. +-type base_key() :: {binary() | [word()], {}}. +-type key(ID) :: {binary() | [word()], {ID}}. +-type nextf() :: fun((key(_) | base_key()) -> key(_)). +-type opts() :: [unique | return_first]. + +%% Holds the constant values of each search. +-record(ctx, { + %% A function which can quickly find the immediate-next record of the given prefix + nextf :: nextf(), + %% The initial prefix to start searching from + %% if the input topic starts with a dollar-word, it's the first word like [<<"$SYS">>] + %% otherwise it's a [] + prefix0 :: [word()], + %% The initial words of a topic + words0 :: [word()], + %% Return as soon as there is one match found + return_first :: boolean() +}). + +%% Holds the variable parts of each search. +-record(acc, { + %% The current searching target topic/filter + target, + %% The number of moves. + %% This is used to check if the target has been moved + %% after attempting to append '+' to the searching prefix + moves = 0, + %% Search result accumulation + matches = [] +}). + +%% All valid utf8 bytes are less than 255. +-define(CEILING_TOPIC, <<255>>). +-define(CEILING, {?CEILING_TOPIC, {1}}). + +%% @doc Return a key which is greater than all other valid keys. +ceiling() -> + ?CEILING. + +%% @doc Make a search-key for the given topic. +-spec make_key(emqx_types:topic(), ID) -> key(ID). +make_key(Topic, ID) when is_binary(Topic) -> + Words = words(Topic), + Key = + case lists:any(fun erlang:is_atom/1, Words) of + true -> + %% it's a wildcard + {Words, {ID}}; + false -> + %% Not a wildcard. We do not split the topic + %% because they can be found with direct lookups. + %% it is also more compact in memory. + {Topic, {ID}} + end, + Key > ceiling() andalso throw({invalid_topic, Topic}), + Key. + +%% @doc Extract record ID from the match. +-spec get_id(key(ID)) -> ID. +get_id({_Filter, {ID}}) -> + ID. + +%% @doc Extract topic (or topic filter) from the match. +-spec get_topic(key(_ID)) -> emqx_types:topic(). +get_topic({Filter, _ID}) when is_list(Filter) -> + emqx_topic:join(Filter); +get_topic({Topic, _ID}) -> + Topic. + +%% Make the base-key which can be used to locate the desired search target. +base(Prefix) -> + {Prefix, {}}. + +%% Move the search target to the key next to the given Base. +move_up(#ctx{nextf = NextF}, #acc{moves = Moves} = Acc, Base) -> + Acc#acc{target = NextF(Base), moves = Moves + 1}. + +%% The current target key is a match, add it to the accumulation. +add(C, #acc{target = Key} = Acc) -> + add(C, Acc, Key). + +%% Add the given key to the accumulation. +add(#ctx{return_first = true}, _Acc, Key) -> + throw({return_first, Key}); +add(_C, #acc{matches = Matches} = Acc, Key) -> + Acc#acc{matches = match_add(Key, Matches)}. + +%% @doc Match given topic against the index and return the first match, or `false` if +%% no match is found. +-spec match(emqx_types:topic(), nextf()) -> false | key(_). +match(Topic, NextF) -> + try search(Topic, NextF, [return_first]) of + [] -> + false + catch + throw:{return_first, Res} -> + Res + end. + +%% @doc Match given topic against the index and return _all_ matches. +%% If `unique` option is given, return only unique matches by record ID. +-spec matches(emqx_types:topic(), nextf(), opts()) -> [key(_)]. +matches(Topic, NextF, Opts) -> + search(Topic, NextF, Opts). + +%% @doc Entrypoint of the search for a given topic. +search(Topic, NextF, Opts) -> + {Words, Prefix} = match_init(Topic), + Context = #ctx{ + nextf = NextF, + prefix0 = Prefix, + words0 = Words, + return_first = proplists:get_bool(return_first, Opts) + }, + Matches0 = + case proplists:get_bool(unique, Opts) of + true -> + #{}; + false -> + [] + end, + Acc = search_new(Context, base(Prefix), #acc{matches = Matches0}), + #acc{matches = Matches} = match_non_wildcards(Context, base(Topic), Acc), + case is_map(Matches) of + true -> + maps:values(Matches); + false -> + Matches + end. + +%% The recursive entrypoint of the trie-search algorithm. +%% Always start from the initial prefix and words. +search_new(#ctx{prefix0 = Prefix, words0 = Words0} = C, NewBase, Acc0) -> + #acc{target = {Filter, _}} = Acc = move_up(C, Acc0, NewBase), + case Prefix of + [] -> + %% This is not a '$' topic, start from '+' + search_plus(C, Words0, Filter, [], Acc); + [DollarWord] -> + %% Start from the '$' word + search_up(C, DollarWord, Words0, Filter, [], Acc) + end. + +%% Search to the bigger end of ordered collection of topics and topic-filters. +search_up(C, Word, Words, Filter, RPrefix, #acc{target = Base} = Acc) -> + case compare(Word, Filter, Words) of + {match, full} -> + search_new(C, Base, add(C, Acc)); + {match, prefix} -> + search_new(C, Base, Acc); + lower -> + Acc; + higher -> + NewBase = base(lists:reverse([Word | RPrefix])), + search_new(C, NewBase, Acc); + shorter -> + search_plus(C, Words, tl(Filter), [Word | RPrefix], Acc) + end. + +%% Try to use '+' as the next word in the prefix. +search_plus(C, [W, X | Words], [W, X | Filter], RPrefix, Acc) -> + %% Directly append the current word to the matching prefix (RPrefix). + %% Micro optimization: try not to call the next clause because + %% it is not a continuation. + search_plus(C, [X | Words], [X | Filter], [W | RPrefix], Acc); +search_plus(C, [W | Words], Filter, RPrefix, Acc) -> + M = Acc#acc.moves, + case search_up(C, '+', Words, Filter, RPrefix, Acc) of + #acc{moves = M1} = Acc1 when M1 =:= M -> + %% Keep searching for one which has W as the next word + search_up(C, W, Words, Filter, RPrefix, Acc1); + Acc1 -> + %% Already searched + Acc1 + end. + +%% Compare prefix word then the next words in suffix against the search-target +%% topic or topic-filter. +compare(_, NotFilter, _) when is_binary(NotFilter) -> + lower; +compare(H, [H | Filter], Words) -> + compare(Filter, Words); +compare(_, ['#'], _Words) -> + {match, full}; +compare(H1, [H2 | _T2], _Words) when H1 < H2 -> + lower; +compare(_H, [_ | _], _Words) -> + higher. + +%% Now compare the filter suffix and the topic suffix. +compare([], []) -> + {match, full}; +compare([], _Words) -> + {match, prefix}; +compare(['#'], _Words) -> + {match, full}; +compare([_ | _], []) -> + lower; +compare([_ | _], _Words) -> + %% cannot know if it's a match, lower, or higher, + %% must search with a longer prefix. + shorter. + +match_add(K = {_Filter, ID}, Acc = #{}) -> + % NOTE: ensuring uniqueness by record ID + Acc#{ID => K}; +match_add(K, Acc) -> + [K | Acc]. + +match_init(Topic) -> + case words(Topic) of + [W = <<"$", _/bytes>> | Rest] -> + % NOTE + % This will effectively skip attempts to match special topics to `#` or `+/...`. + {Rest, [W]}; + Words -> + {Words, []} + end. + +-spec words(emqx_types:topic()) -> [word()]. +words(Topic) when is_binary(Topic) -> + % NOTE + % This is almost identical to `emqx_topic:words/1`, but it doesn't convert empty + % tokens to ''. This is needed to keep ordering of words consistent with what + % `match_filter/3` expects. + [word(W) || W <- emqx_topic:tokens(Topic)]. + +-spec word(binary()) -> word(). +word(<<"+">>) -> '+'; +word(<<"#">>) -> '#'; +word(Bin) -> Bin. + +match_non_wildcards(#ctx{nextf = NextF} = C, {Topic, _} = Base, Acc) -> + case NextF(Base) of + {Topic, _ID} = Key -> + match_non_wildcards(C, Key, add(C, Acc, Key)); + _Other -> + Acc + end. diff --git a/apps/emqx/test/emqx_topic_index_SUITE.erl b/apps/emqx/test/emqx_topic_index_SUITE.erl index 9058f3597..dd1fc07f3 100644 --- a/apps/emqx/test/emqx_topic_index_SUITE.erl +++ b/apps/emqx/test/emqx_topic_index_SUITE.erl @@ -25,42 +25,71 @@ -import(emqx_proper_types, [scaled/2]). all() -> - emqx_common_test_helpers:all(?MODULE). + [ + {group, ets}, + {group, gb_tree} + ]. -t_insert(_) -> - Tab = emqx_topic_index:new(), - true = emqx_topic_index:insert(<<"sensor/1/metric/2">>, t_insert_1, <<>>, Tab), - true = emqx_topic_index:insert(<<"sensor/+/#">>, t_insert_2, <<>>, Tab), - true = emqx_topic_index:insert(<<"sensor/#">>, t_insert_3, <<>>, Tab), - ?assertEqual(<<"sensor/#">>, topic(match(<<"sensor">>, Tab))), - ?assertEqual(t_insert_3, id(match(<<"sensor">>, Tab))). +groups() -> + All = emqx_common_test_helpers:all(?MODULE), + [ + {ets, All}, + {gb_tree, All} + ]. -t_match(_) -> - Tab = emqx_topic_index:new(), - true = emqx_topic_index:insert(<<"sensor/1/metric/2">>, t_match_1, <<>>, Tab), - true = emqx_topic_index:insert(<<"sensor/+/#">>, t_match_2, <<>>, Tab), - true = emqx_topic_index:insert(<<"sensor/#">>, t_match_3, <<>>, Tab), +init_per_group(ets, Config) -> + [{index_module, emqx_topic_index} | Config]; +init_per_group(gb_tree, Config) -> + [{index_module, emqx_topic_gbt} | Config]. + +end_per_group(_Group, _Config) -> + ok. + +get_module(Config) -> + proplists:get_value(index_module, Config). + +t_insert(Config) -> + M = get_module(Config), + Tab = M:new(), + true = M:insert(<<"sensor/1/metric/2">>, t_insert_1, <<>>, Tab), + true = M:insert(<<"sensor/+/#">>, t_insert_2, <<>>, Tab), + true = M:insert(<<"sensor/#">>, t_insert_3, <<>>, Tab), + ?assertEqual(<<"sensor/#">>, topic(match(M, <<"sensor">>, Tab))), + ?assertEqual(t_insert_3, id(match(M, <<"sensor">>, Tab))). + +t_match(Config) -> + M = get_module(Config), + Tab = M:new(), + true = M:insert(<<"sensor/1/metric/2">>, t_match_1, <<>>, Tab), + true = M:insert(<<"sensor/+/#">>, t_match_2, <<>>, Tab), + true = M:insert(<<"sensor/#">>, t_match_3, <<>>, Tab), ?assertMatch( [<<"sensor/#">>, <<"sensor/+/#">>], - [topic(M) || M <- matches(<<"sensor/1">>, Tab)] + [topic(X) || X <- matches(M, <<"sensor/1">>, Tab)] ). -t_match2(_) -> - Tab = emqx_topic_index:new(), - true = emqx_topic_index:insert(<<"#">>, t_match2_1, <<>>, Tab), - true = emqx_topic_index:insert(<<"+/#">>, t_match2_2, <<>>, Tab), - true = emqx_topic_index:insert(<<"+/+/#">>, t_match2_3, <<>>, Tab), +t_match2(Config) -> + M = get_module(Config), + Tab = M:new(), + true = M:insert(<<"#">>, t_match2_1, <<>>, Tab), + true = M:insert(<<"+/#">>, t_match2_2, <<>>, Tab), + true = M:insert(<<"+/+/#">>, t_match2_3, <<>>, Tab), ?assertEqual( [<<"#">>, <<"+/#">>, <<"+/+/#">>], - [topic(M) || M <- matches(<<"a/b/c">>, Tab)] + [topic(X) || X <- matches(M, <<"a/b/c">>, Tab)] ), ?assertEqual( false, - emqx_topic_index:match(<<"$SYS/broker/zenmq">>, Tab) + M:match(<<"$SYS/broker/zenmq">>, Tab) + ), + ?assertEqual( + [], + matches(M, <<"$SYS/broker/zenmq">>, Tab) ). -t_match3(_) -> - Tab = emqx_topic_index:new(), +t_match3(Config) -> + M = get_module(Config), + Tab = M:new(), Records = [ {<<"d/#">>, t_match3_1}, {<<"a/b/+">>, t_match3_2}, @@ -69,37 +98,39 @@ t_match3(_) -> {<<"$SYS/#">>, t_match3_sys} ], lists:foreach( - fun({Topic, ID}) -> emqx_topic_index:insert(Topic, ID, <<>>, Tab) end, + fun({Topic, ID}) -> M:insert(Topic, ID, <<>>, Tab) end, Records ), - Matched = matches(<<"a/b/c">>, Tab), + Matched = matches(M, <<"a/b/c">>, Tab), case length(Matched) of 3 -> ok; _ -> error({unexpected, Matched}) end, ?assertEqual( t_match3_sys, - id(match(<<"$SYS/a/b/c">>, Tab)) + id(match(M, <<"$SYS/a/b/c">>, Tab)) ). -t_match4(_) -> - Tab = emqx_topic_index:new(), +t_match4(Config) -> + M = get_module(Config), + Tab = M:new(), Records = [{<<"/#">>, t_match4_1}, {<<"/+">>, t_match4_2}, {<<"/+/a/b/c">>, t_match4_3}], lists:foreach( - fun({Topic, ID}) -> emqx_topic_index:insert(Topic, ID, <<>>, Tab) end, + fun({Topic, ID}) -> M:insert(Topic, ID, <<>>, Tab) end, Records ), ?assertEqual( [<<"/#">>, <<"/+">>], - [topic(M) || M <- matches(<<"/">>, Tab)] + [topic(X) || X <- matches(M, <<"/">>, Tab)] ), ?assertEqual( [<<"/#">>, <<"/+/a/b/c">>], - [topic(M) || M <- matches(<<"/0/a/b/c">>, Tab)] + [topic(X) || X <- matches(M, <<"/0/a/b/c">>, Tab)] ). -t_match5(_) -> - Tab = emqx_topic_index:new(), +t_match5(Config) -> + M = get_module(Config), + Tab = M:new(), T = <<"a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z">>, Records = [ {<<"#">>, t_match5_1}, @@ -107,58 +138,63 @@ t_match5(_) -> {<>, t_match5_3} ], lists:foreach( - fun({Topic, ID}) -> emqx_topic_index:insert(Topic, ID, <<>>, Tab) end, + fun({Topic, ID}) -> M:insert(Topic, ID, <<>>, Tab) end, Records ), ?assertEqual( [<<"#">>, <>], - [topic(M) || M <- matches(T, Tab)] + [topic(X) || X <- matches(M, T, Tab)] ), ?assertEqual( [<<"#">>, <>, <>], - [topic(M) || M <- matches(<>, Tab)] + [topic(X) || X <- matches(M, <>, Tab)] ). -t_match6(_) -> - Tab = emqx_topic_index:new(), +t_match6(Config) -> + M = get_module(Config), + Tab = M:new(), T = <<"a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z">>, W = <<"+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/#">>, - emqx_topic_index:insert(W, ID = t_match6, <<>>, Tab), - ?assertEqual(ID, id(match(T, Tab))). + M:insert(W, ID = t_match6, <<>>, Tab), + ?assertEqual(ID, id(match(M, T, Tab))). -t_match7(_) -> - Tab = emqx_topic_index:new(), +t_match7(Config) -> + M = get_module(Config), + Tab = M:new(), T = <<"a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z">>, W = <<"a/+/c/+/e/+/g/+/i/+/k/+/m/+/o/+/q/+/s/+/u/+/w/+/y/+/#">>, - emqx_topic_index:insert(W, t_match7, <<>>, Tab), - ?assertEqual(W, topic(match(T, Tab))). + M:insert(W, t_match7, <<>>, Tab), + ?assertEqual(W, topic(match(M, T, Tab))). -t_match_fast_forward(_) -> - Tab = emqx_topic_index:new(), - emqx_topic_index:insert(<<"a/b/1/2/3/4/5/6/7/8/9/#">>, id1, <<>>, Tab), - emqx_topic_index:insert(<<"z/y/x/+/+">>, id2, <<>>, Tab), - emqx_topic_index:insert(<<"a/b/c/+">>, id3, <<>>, Tab), +t_match_fast_forward(Config) -> + M = get_module(Config), + Tab = M:new(), + M:insert(<<"a/b/1/2/3/4/5/6/7/8/9/#">>, id1, <<>>, Tab), + M:insert(<<"z/y/x/+/+">>, id2, <<>>, Tab), + M:insert(<<"a/b/c/+">>, id3, <<>>, Tab), % dbg:tracer(), % dbg:p(all, c), % dbg:tpl({ets, next, '_'}, x), - ?assertEqual(id1, id(match(<<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab))), - ?assertEqual([id1], [id(M) || M <- matches(<<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab)]). + ?assertEqual(id1, id(match(M, <<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab))), + ?assertEqual([id1], [id(X) || X <- matches(M, <<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab)]). -t_match_unique(_) -> - Tab = emqx_topic_index:new(), - emqx_topic_index:insert(<<"a/b/c">>, t_match_id1, <<>>, Tab), - emqx_topic_index:insert(<<"a/b/+">>, t_match_id1, <<>>, Tab), - emqx_topic_index:insert(<<"a/b/c/+">>, t_match_id2, <<>>, Tab), +t_match_unique(Config) -> + M = get_module(Config), + Tab = M:new(), + M:insert(<<"a/b/c">>, t_match_id1, <<>>, Tab), + M:insert(<<"a/b/+">>, t_match_id1, <<>>, Tab), + M:insert(<<"a/b/c/+">>, t_match_id2, <<>>, Tab), ?assertEqual( [t_match_id1, t_match_id1], - [id(M) || M <- emqx_topic_index:matches(<<"a/b/c">>, Tab, [])] + [id(X) || X <- matches(M, <<"a/b/c">>, Tab, [])] ), ?assertEqual( [t_match_id1], - [id(M) || M <- emqx_topic_index:matches(<<"a/b/c">>, Tab, [unique])] + [id(X) || X <- matches(M, <<"a/b/c">>, Tab, [unique])] ). -t_match_wildcard_edge_cases(_) -> +t_match_wildcard_edge_cases(Config) -> + M = get_module(Config), CommonTopics = [ <<"a/b">>, <<"a/b/#">>, @@ -179,32 +215,33 @@ t_match_wildcard_edge_cases(_) -> {[<<"/">>, <<"+">>], <<"a">>, [2]} ], F = fun({Topics, TopicName, Expected}) -> - Tab = emqx_topic_index:new(), - _ = [emqx_topic_index:insert(T, N, <<>>, Tab) || {N, T} <- lists:enumerate(Topics)], + Tab = M:new(), + _ = [M:insert(T, N, <<>>, Tab) || {N, T} <- lists:enumerate(Topics)], ?assertEqual( lists:last(Expected), - id(emqx_topic_index:match(TopicName, Tab)), + id(M:match(TopicName, Tab)), #{"Base topics" => Topics, "Topic name" => TopicName} ), ?assertEqual( Expected, - [id(M) || M <- emqx_topic_index:matches(TopicName, Tab, [unique])], + [id(X) || X <- matches(M, TopicName, Tab, [unique])], #{"Base topics" => Topics, "Topic name" => TopicName} ) end, lists:foreach(F, Datasets). -t_prop_matches(_) -> +t_prop_matches(Config) -> + M = get_module(Config), ?assert( proper:quickcheck( - topic_matches_prop(), + topic_matches_prop(M), [{max_size, 100}, {numtests, 100}] ) ), Statistics = [{C, account(C)} || C <- [filters, topics, matches, maxhits]], ct:pal("Statistics: ~p", [maps:from_list(Statistics)]). -topic_matches_prop() -> +topic_matches_prop(M) -> ?FORALL( % Generate a longer list of topics and a shorter list of topic filter patterns. #{ @@ -219,12 +256,12 @@ topic_matches_prop() -> patterns => list(topic_filter_pattern_t()) }), begin - Tab = emqx_topic_index:new(), + Tab = M:new(), Topics = [emqx_topic:join(T) || T <- TTopics], % Produce topic filters from generated topics and patterns. % Number of filters is equal to the number of patterns, most of the time. Filters = lists:enumerate(mk_filters(Pats, TTopics)), - _ = [emqx_topic_index:insert(F, N, <<>>, Tab) || {N, F} <- Filters], + _ = [M:insert(F, N, <<>>, Tab) || {N, F} <- Filters], % Gather some basic statistics _ = account(filters, length(Filters)), _ = account(topics, NTopics = length(Topics)), @@ -233,7 +270,7 @@ topic_matches_prop() -> % matching it against the list of filters one by one. lists:all( fun(Topic) -> - Ids1 = [id(M) || M <- emqx_topic_index:matches(Topic, Tab, [unique])], + Ids1 = [id(X) || X <- matches(M, Topic, Tab, [unique])], Ids2 = lists:filtermap( fun({N, F}) -> case emqx_topic:match(Topic, F) of @@ -277,17 +314,20 @@ account(Counter) -> %% -match(T, Tab) -> - emqx_topic_index:match(T, Tab). +match(M, T, Tab) -> + M:match(T, Tab). -matches(T, Tab) -> - lists:sort(emqx_topic_index:matches(T, Tab, [])). +matches(M, T, Tab) -> + lists:sort(M:matches(T, Tab, [])). + +matches(M, T, Tab, Opts) -> + M:matches(T, Tab, Opts). id(Match) -> - emqx_topic_index:get_id(Match). + emqx_trie_search:get_id(Match). topic(Match) -> - emqx_topic_index:get_topic(Match). + emqx_trie_search:get_topic(Match). %% From a1e66356147245a57ba1fc58d888927cbfdd3093 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 23 Aug 2023 10:12:46 +0200 Subject: [PATCH 051/119] refactor(topic_index): no forced ceiling entry in index table --- apps/emqx/src/emqx_topic_gbt.erl | 10 +++++-- apps/emqx/src/emqx_topic_index.erl | 4 +-- apps/emqx/src/emqx_trie_search.erl | 45 ++++++++++++------------------ 3 files changed, 26 insertions(+), 33 deletions(-) diff --git a/apps/emqx/src/emqx_topic_gbt.erl b/apps/emqx/src/emqx_topic_gbt.erl index da6320bc7..dade70cee 100644 --- a/apps/emqx/src/emqx_topic_gbt.erl +++ b/apps/emqx/src/emqx_topic_gbt.erl @@ -104,12 +104,16 @@ gbt_update(Name, Tree) -> true. gbt_next(nil, _Input) -> - emqx_trie_search:ceiling(); + '$end_of_table'; gbt_next({P, _V, _Smaller, Bigger}, K) when K >= P -> gbt_next(Bigger, K); gbt_next({P, _V, Smaller, _Bigger}, K) -> - NextKey = gbt_next(Smaller, K), - min(P, NextKey). + case gbt_next(Smaller, K) of + '$end_of_table' -> + P; + NextKey -> + NextKey + end. make_nextf(Name) -> {_SizeWeDontCare, TheTree} = gbt(Name), diff --git a/apps/emqx/src/emqx_topic_index.erl b/apps/emqx/src/emqx_topic_index.erl index e1da5b3b7..acbf36627 100644 --- a/apps/emqx/src/emqx_topic_index.erl +++ b/apps/emqx/src/emqx_topic_index.erl @@ -35,9 +35,7 @@ %% Usable mostly for testing purposes. -spec new() -> ets:table(). new() -> - T = ets:new(?MODULE, [public, ordered_set, {read_concurrency, true}]), - ets:insert(T, {emqx_trie_search:ceiling(), []}), - T. + ets:new(?MODULE, [public, ordered_set, {read_concurrency, true}]). %% @doc Insert a new entry into the index that associates given topic filter to given %% record ID, and attaches arbitrary record to the entry. This allows users to choose diff --git a/apps/emqx/src/emqx_trie_search.erl b/apps/emqx/src/emqx_trie_search.erl index 8d9463c26..a25f3c9f4 100644 --- a/apps/emqx/src/emqx_trie_search.erl +++ b/apps/emqx/src/emqx_trie_search.erl @@ -98,14 +98,14 @@ -module(emqx_trie_search). --export([ceiling/0, make_key/2]). +-export([make_key/2]). -export([match/2, matches/3, get_id/1, get_topic/1]). -export_type([key/1, word/0, nextf/0, opts/0]). -type word() :: binary() | '+' | '#'. -type base_key() :: {binary() | [word()], {}}. -type key(ID) :: {binary() | [word()], {ID}}. --type nextf() :: fun((key(_) | base_key()) -> key(_)). +-type nextf() :: fun((key(_) | base_key()) -> '$end_of_table' | key(_)). -type opts() :: [unique | return_first]. %% Holds the constant values of each search. @@ -134,31 +134,20 @@ matches = [] }). -%% All valid utf8 bytes are less than 255. --define(CEILING_TOPIC, <<255>>). --define(CEILING, {?CEILING_TOPIC, {1}}). - -%% @doc Return a key which is greater than all other valid keys. -ceiling() -> - ?CEILING. - %% @doc Make a search-key for the given topic. -spec make_key(emqx_types:topic(), ID) -> key(ID). make_key(Topic, ID) when is_binary(Topic) -> Words = words(Topic), - Key = - case lists:any(fun erlang:is_atom/1, Words) of - true -> - %% it's a wildcard - {Words, {ID}}; - false -> - %% Not a wildcard. We do not split the topic - %% because they can be found with direct lookups. - %% it is also more compact in memory. - {Topic, {ID}} - end, - Key > ceiling() andalso throw({invalid_topic, Topic}), - Key. + case lists:any(fun erlang:is_atom/1, Words) of + true -> + %% it's a wildcard + {Words, {ID}}; + false -> + %% Not a wildcard. We do not split the topic + %% because they can be found with direct lookups. + %% it is also more compact in memory. + {Topic, {ID}} + end. %% @doc Extract record ID from the match. -spec get_id(key(ID)) -> ID. @@ -236,12 +225,14 @@ search(Topic, NextF, Opts) -> %% The recursive entrypoint of the trie-search algorithm. %% Always start from the initial prefix and words. search_new(#ctx{prefix0 = Prefix, words0 = Words0} = C, NewBase, Acc0) -> - #acc{target = {Filter, _}} = Acc = move_up(C, Acc0, NewBase), - case Prefix of - [] -> + case move_up(C, Acc0, NewBase) of + #acc{target = '$end_of_table'} = Acc -> + Acc; + #acc{target = {Filter, _}} = Acc when Prefix =:= [] -> %% This is not a '$' topic, start from '+' search_plus(C, Words0, Filter, [], Acc); - [DollarWord] -> + #acc{target = {Filter, _}} = Acc -> + [DollarWord] = Prefix, %% Start from the '$' word search_up(C, DollarWord, Words0, Filter, [], Acc) end. From a30d87e14f3ab6ce7635159c47df1275861d9598 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 24 Aug 2023 11:51:34 +0200 Subject: [PATCH 052/119] refactor(topic_index): remove more unnecessary next calls also avoid using records (setelement) for recursive return values --- apps/emqx/src/emqx_trie_search.erl | 119 ++++++++++------------ apps/emqx/test/emqx_topic_index_SUITE.erl | 15 +++ 2 files changed, 71 insertions(+), 63 deletions(-) diff --git a/apps/emqx/src/emqx_trie_search.erl b/apps/emqx/src/emqx_trie_search.erl index a25f3c9f4..36ccd656b 100644 --- a/apps/emqx/src/emqx_trie_search.erl +++ b/apps/emqx/src/emqx_trie_search.erl @@ -102,12 +102,15 @@ -export([match/2, matches/3, get_id/1, get_topic/1]). -export_type([key/1, word/0, nextf/0, opts/0]). +-define(END, '$end_of_table'). + -type word() :: binary() | '+' | '#'. -type base_key() :: {binary() | [word()], {}}. -type key(ID) :: {binary() | [word()], {ID}}. --type nextf() :: fun((key(_) | base_key()) -> '$end_of_table' | key(_)). +-type nextf() :: fun((key(_) | base_key()) -> ?END | key(_)). -type opts() :: [unique | return_first]. + %% Holds the constant values of each search. -record(ctx, { %% A function which can quickly find the immediate-next record of the given prefix @@ -122,18 +125,6 @@ return_first :: boolean() }). -%% Holds the variable parts of each search. --record(acc, { - %% The current searching target topic/filter - target, - %% The number of moves. - %% This is used to check if the target has been moved - %% after attempting to append '+' to the searching prefix - moves = 0, - %% Search result accumulation - matches = [] -}). - %% @doc Make a search-key for the given topic. -spec make_key(emqx_types:topic(), ID) -> key(ID). make_key(Topic, ID) when is_binary(Topic) -> @@ -166,18 +157,14 @@ base(Prefix) -> {Prefix, {}}. %% Move the search target to the key next to the given Base. -move_up(#ctx{nextf = NextF}, #acc{moves = Moves} = Acc, Base) -> - Acc#acc{target = NextF(Base), moves = Moves + 1}. - -%% The current target key is a match, add it to the accumulation. -add(C, #acc{target = Key} = Acc) -> - add(C, Acc, Key). +move_up(#ctx{nextf = NextF}, Base) -> + NextF(Base). %% Add the given key to the accumulation. add(#ctx{return_first = true}, _Acc, Key) -> throw({return_first, Key}); -add(_C, #acc{matches = Matches} = Acc, Key) -> - Acc#acc{matches = match_add(Key, Matches)}. +add(_C, Acc, Key) -> + match_add(Key, Acc). %% @doc Match given topic against the index and return the first match, or `false` if %% no match is found. @@ -206,69 +193,69 @@ search(Topic, NextF, Opts) -> words0 = Words, return_first = proplists:get_bool(return_first, Opts) }, - Matches0 = + Acc0 = case proplists:get_bool(unique, Opts) of true -> #{}; false -> [] end, - Acc = search_new(Context, base(Prefix), #acc{matches = Matches0}), - #acc{matches = Matches} = match_non_wildcards(Context, base(Topic), Acc), - case is_map(Matches) of + {MaybeEnd, Acc1} = search_new(Context, base(Prefix), Acc0), + Acc = match_topics(Context, Topic, MaybeEnd, Acc1), + case is_map(Acc) of true -> - maps:values(Matches); + maps:values(Acc); false -> - Matches + Acc end. %% The recursive entrypoint of the trie-search algorithm. %% Always start from the initial prefix and words. -search_new(#ctx{prefix0 = Prefix, words0 = Words0} = C, NewBase, Acc0) -> - case move_up(C, Acc0, NewBase) of - #acc{target = '$end_of_table'} = Acc -> - Acc; - #acc{target = {Filter, _}} = Acc when Prefix =:= [] -> - %% This is not a '$' topic, start from '+' - search_plus(C, Words0, Filter, [], Acc); - #acc{target = {Filter, _}} = Acc -> - [DollarWord] = Prefix, - %% Start from the '$' word - search_up(C, DollarWord, Words0, Filter, [], Acc) - end. +search_new(C, NewBase, Acc) -> + search_moved(C, move_up(C, NewBase), Acc). + +search_moved(_, ?END, Acc) -> + {?END, Acc}; +search_moved(#ctx{prefix0 = [], words0 = Words0} = C, {Filter, _} = T, Acc) -> + %% This is not a '$' topic, start from '+' + search_plus(C, Words0, Filter, [], T, Acc); +search_moved(#ctx{prefix0 = Prefix, words0 = Words0} = C, {Filter, _} = T, Acc) -> + [DollarWord] = Prefix, + %% Start from the '$' word + search_up(C, DollarWord, Words0, Filter, [], T, Acc). %% Search to the bigger end of ordered collection of topics and topic-filters. -search_up(C, Word, Words, Filter, RPrefix, #acc{target = Base} = Acc) -> +search_up(C, Word, Words, Filter, RPrefix, T, Acc) -> case compare(Word, Filter, Words) of {match, full} -> - search_new(C, Base, add(C, Acc)); + search_new(C, T, add(C, Acc, T)); {match, prefix} -> - search_new(C, Base, Acc); + search_new(C, T, Acc); lower -> - Acc; + {T, Acc}; higher -> NewBase = base(lists:reverse([Word | RPrefix])), search_new(C, NewBase, Acc); shorter -> - search_plus(C, Words, tl(Filter), [Word | RPrefix], Acc) + search_plus(C, Words, tl(Filter), [Word | RPrefix], T, Acc) end. %% Try to use '+' as the next word in the prefix. -search_plus(C, [W, X | Words], [W, X | Filter], RPrefix, Acc) -> +search_plus(C, [W, X | Words], [W, X | Filter], RPrefix, T, Acc) -> %% Directly append the current word to the matching prefix (RPrefix). %% Micro optimization: try not to call the next clause because %% it is not a continuation. - search_plus(C, [X | Words], [X | Filter], [W | RPrefix], Acc); -search_plus(C, [W | Words], Filter, RPrefix, Acc) -> - M = Acc#acc.moves, - case search_up(C, '+', Words, Filter, RPrefix, Acc) of - #acc{moves = M1} = Acc1 when M1 =:= M -> - %% Keep searching for one which has W as the next word - search_up(C, W, Words, Filter, RPrefix, Acc1); - Acc1 -> - %% Already searched - Acc1 - end. + search_plus(C, [X | Words], [X | Filter], [W | RPrefix], T, Acc); +search_plus(C, [W | Words], ['+' | _] = Filter, RPrefix, T, Acc) -> + case search_up(C, '+', Words, Filter, RPrefix, T, Acc) of + {T, Acc} -> + search_up(C, W, Words, Filter, RPrefix, T, Acc); + TargetMoved -> + TargetMoved + end; +search_plus(C, [W | Words], Filter, RPrefix, T, Acc) -> + %% not a plus + search_up(C, W, Words, Filter, RPrefix, T, Acc). %% Compare prefix word then the next words in suffix against the search-target %% topic or topic-filter. @@ -326,10 +313,16 @@ word(<<"+">>) -> '+'; word(<<"#">>) -> '#'; word(Bin) -> Bin. -match_non_wildcards(#ctx{nextf = NextF} = C, {Topic, _} = Base, Acc) -> - case NextF(Base) of - {Topic, _ID} = Key -> - match_non_wildcards(C, Key, add(C, Acc, Key)); - _Other -> - Acc - end. +%% match non-wildcard topics +match_topics(#ctx{nextf = NextF} = C, Topic, {Topic, _} = Key, Acc) -> + %% found a topic match + match_topics(C, Topic, NextF(Key), add(C, Acc, Key)); +match_topics(#ctx{nextf = NextF} = C, Topic, {F, _}, Acc) when F < Topic -> + %% the last key is a filter, try jump to the topic + match_topics(C, Topic, NextF(base(Topic)), Acc); +match_topics(#ctx{nextf = NextF} = C, Topic, continue, Acc) -> + %% the last key is a '+/...' wildcard + match_topics(C, Topic, NextF(base(Topic)), Acc); +match_topics(_C, _Topic, _Key, Acc) -> + %% gone pass the topic + Acc. diff --git a/apps/emqx/test/emqx_topic_index_SUITE.erl b/apps/emqx/test/emqx_topic_index_SUITE.erl index dd1fc07f3..eb0ac0e40 100644 --- a/apps/emqx/test/emqx_topic_index_SUITE.erl +++ b/apps/emqx/test/emqx_topic_index_SUITE.erl @@ -166,6 +166,21 @@ t_match7(Config) -> M:insert(W, t_match7, <<>>, Tab), ?assertEqual(W, topic(match(M, T, Tab))). +t_match8(Config) -> + M = get_module(Config), + Tab = M:new(), + Filters = [<<"+">>, <<"dev/global/sensor">>, <<"dev/+/sensor/#">>], + IDs = [1,2,3], + Keys = [{F, ID} || F <- Filters, ID <- IDs], + lists:foreach(fun({F, ID}) -> + M:insert(F, ID, <<>>, Tab) + end, Keys), + Topic = <<"dev/global/sensor">>, + Matches = lists:sort(matches(M, Topic, Tab)), + ?assertEqual([<<"dev/+/sensor/#">>, <<"dev/+/sensor/#">>, <<"dev/+/sensor/#">>, + <<"dev/global/sensor">>, <<"dev/global/sensor">>, <<"dev/global/sensor">>], + [emqx_topic_index:get_topic(Match) || Match <- Matches]). + t_match_fast_forward(Config) -> M = get_module(Config), Tab = M:new(), From 62423b0b12deac9641152a5ff01236660fbf3746 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 24 Aug 2023 13:08:24 +0200 Subject: [PATCH 053/119] refactor(topic_index): less special handling for leading $ words --- apps/emqx/src/emqx_trie_search.erl | 92 ++++++++++------------- apps/emqx/test/emqx_topic_index_SUITE.erl | 25 ++++-- 2 files changed, 56 insertions(+), 61 deletions(-) diff --git a/apps/emqx/src/emqx_trie_search.erl b/apps/emqx/src/emqx_trie_search.erl index 36ccd656b..71bc8d8a0 100644 --- a/apps/emqx/src/emqx_trie_search.erl +++ b/apps/emqx/src/emqx_trie_search.erl @@ -16,10 +16,10 @@ %% @doc Topic index for matching topics to topic filters. %% -%% Works on top of ETS ordered_set table. Keys are tuples constructed from -%% parsed topic filters and record IDs, wrapped in a tuple to order them -%% strictly greater than unit tuple (`{}`). Existing table may be used if -%% existing keys will not collide with index keys. +%% Works on top of a ordered collection data set, such as ETS ordered_set table. +%% Keys are tuples constructed from parsed topic filters and record IDs, +%% wrapped in a tuple to order them strictly greater than unit tuple (`{}`). +%% Existing table may be used if existing keys will not collide with index keys. %% %% Designed to effectively answer questions like: %% 1. Does any topic filter match given topic? @@ -110,15 +110,10 @@ -type nextf() :: fun((key(_) | base_key()) -> ?END | key(_)). -type opts() :: [unique | return_first]. - %% Holds the constant values of each search. -record(ctx, { %% A function which can quickly find the immediate-next record of the given prefix nextf :: nextf(), - %% The initial prefix to start searching from - %% if the input topic starts with a dollar-word, it's the first word like [<<"$SYS">>] - %% otherwise it's a [] - prefix0 :: [word()], %% The initial words of a topic words0 :: [word()], %% Return as soon as there is one match found @@ -129,7 +124,7 @@ -spec make_key(emqx_types:topic(), ID) -> key(ID). make_key(Topic, ID) when is_binary(Topic) -> Words = words(Topic), - case lists:any(fun erlang:is_atom/1, Words) of + case emqx_topic:wildcard(Words) of true -> %% it's a wildcard {Words, {ID}}; @@ -186,10 +181,9 @@ matches(Topic, NextF, Opts) -> %% @doc Entrypoint of the search for a given topic. search(Topic, NextF, Opts) -> - {Words, Prefix} = match_init(Topic), + Words = words(Topic), Context = #ctx{ nextf = NextF, - prefix0 = Prefix, words0 = Words, return_first = proplists:get_bool(return_first, Opts) }, @@ -200,7 +194,15 @@ search(Topic, NextF, Opts) -> false -> [] end, - {MaybeEnd, Acc1} = search_new(Context, base(Prefix), Acc0), + Base = + case hd(Words) of + <<$$, _/binary>> -> + %% skip all filters starts with # or + + base([hd(Words)]); + _ -> + base([]) + end, + {MaybeEnd, Acc1} = search_new(Context, Base, Acc0), Acc = match_topics(Context, Topic, MaybeEnd, Acc1), case is_map(Acc) of true -> @@ -211,18 +213,30 @@ search(Topic, NextF, Opts) -> %% The recursive entrypoint of the trie-search algorithm. %% Always start from the initial prefix and words. -search_new(C, NewBase, Acc) -> - search_moved(C, move_up(C, NewBase), Acc). +search_new(#ctx{words0 = Words} = C, NewBase, Acc) -> + case move_up(C, NewBase) of + ?END -> + {?END, Acc}; + {Filter, _} = T -> + search_plus(C, Words, Filter, [], T, Acc) + end. -search_moved(_, ?END, Acc) -> - {?END, Acc}; -search_moved(#ctx{prefix0 = [], words0 = Words0} = C, {Filter, _} = T, Acc) -> - %% This is not a '$' topic, start from '+' - search_plus(C, Words0, Filter, [], T, Acc); -search_moved(#ctx{prefix0 = Prefix, words0 = Words0} = C, {Filter, _} = T, Acc) -> - [DollarWord] = Prefix, - %% Start from the '$' word - search_up(C, DollarWord, Words0, Filter, [], T, Acc). +%% Try to use '+' as the next word in the prefix. +search_plus(C, [W, X | Words], [W, X | Filter], RPrefix, T, Acc) -> + %% Directly append the current word to the matching prefix (RPrefix). + %% Micro optimization: try not to call the next clause because + %% it is not a continuation. + search_plus(C, [X | Words], [X | Filter], [W | RPrefix], T, Acc); +search_plus(C, [W | Words], ['+' | _] = Filter, RPrefix, T, Acc) -> + case search_up(C, '+', Words, Filter, RPrefix, T, Acc) of + {T, Acc1} -> + search_up(C, W, Words, Filter, RPrefix, T, Acc1); + TargetMoved -> + TargetMoved + end; +search_plus(C, [W | Words], Filter, RPrefix, T, Acc) -> + %% not a plus + search_up(C, W, Words, Filter, RPrefix, T, Acc). %% Search to the bigger end of ordered collection of topics and topic-filters. search_up(C, Word, Words, Filter, RPrefix, T, Acc) -> @@ -240,23 +254,6 @@ search_up(C, Word, Words, Filter, RPrefix, T, Acc) -> search_plus(C, Words, tl(Filter), [Word | RPrefix], T, Acc) end. -%% Try to use '+' as the next word in the prefix. -search_plus(C, [W, X | Words], [W, X | Filter], RPrefix, T, Acc) -> - %% Directly append the current word to the matching prefix (RPrefix). - %% Micro optimization: try not to call the next clause because - %% it is not a continuation. - search_plus(C, [X | Words], [X | Filter], [W | RPrefix], T, Acc); -search_plus(C, [W | Words], ['+' | _] = Filter, RPrefix, T, Acc) -> - case search_up(C, '+', Words, Filter, RPrefix, T, Acc) of - {T, Acc} -> - search_up(C, W, Words, Filter, RPrefix, T, Acc); - TargetMoved -> - TargetMoved - end; -search_plus(C, [W | Words], Filter, RPrefix, T, Acc) -> - %% not a plus - search_up(C, W, Words, Filter, RPrefix, T, Acc). - %% Compare prefix word then the next words in suffix against the search-target %% topic or topic-filter. compare(_, NotFilter, _) when is_binary(NotFilter) -> @@ -290,16 +287,6 @@ match_add(K = {_Filter, ID}, Acc = #{}) -> match_add(K, Acc) -> [K | Acc]. -match_init(Topic) -> - case words(Topic) of - [W = <<"$", _/bytes>> | Rest] -> - % NOTE - % This will effectively skip attempts to match special topics to `#` or `+/...`. - {Rest, [W]}; - Words -> - {Words, []} - end. - -spec words(emqx_types:topic()) -> [word()]. words(Topic) when is_binary(Topic) -> % NOTE @@ -320,9 +307,6 @@ match_topics(#ctx{nextf = NextF} = C, Topic, {Topic, _} = Key, Acc) -> match_topics(#ctx{nextf = NextF} = C, Topic, {F, _}, Acc) when F < Topic -> %% the last key is a filter, try jump to the topic match_topics(C, Topic, NextF(base(Topic)), Acc); -match_topics(#ctx{nextf = NextF} = C, Topic, continue, Acc) -> - %% the last key is a '+/...' wildcard - match_topics(C, Topic, NextF(base(Topic)), Acc); match_topics(_C, _Topic, _Key, Acc) -> %% gone pass the topic Acc. diff --git a/apps/emqx/test/emqx_topic_index_SUITE.erl b/apps/emqx/test/emqx_topic_index_SUITE.erl index eb0ac0e40..551cc2438 100644 --- a/apps/emqx/test/emqx_topic_index_SUITE.erl +++ b/apps/emqx/test/emqx_topic_index_SUITE.erl @@ -170,16 +170,27 @@ t_match8(Config) -> M = get_module(Config), Tab = M:new(), Filters = [<<"+">>, <<"dev/global/sensor">>, <<"dev/+/sensor/#">>], - IDs = [1,2,3], + IDs = [1, 2, 3], Keys = [{F, ID} || F <- Filters, ID <- IDs], - lists:foreach(fun({F, ID}) -> - M:insert(F, ID, <<>>, Tab) - end, Keys), + lists:foreach( + fun({F, ID}) -> + M:insert(F, ID, <<>>, Tab) + end, + Keys + ), Topic = <<"dev/global/sensor">>, Matches = lists:sort(matches(M, Topic, Tab)), - ?assertEqual([<<"dev/+/sensor/#">>, <<"dev/+/sensor/#">>, <<"dev/+/sensor/#">>, - <<"dev/global/sensor">>, <<"dev/global/sensor">>, <<"dev/global/sensor">>], - [emqx_topic_index:get_topic(Match) || Match <- Matches]). + ?assertEqual( + [ + <<"dev/+/sensor/#">>, + <<"dev/+/sensor/#">>, + <<"dev/+/sensor/#">>, + <<"dev/global/sensor">>, + <<"dev/global/sensor">>, + <<"dev/global/sensor">> + ], + [emqx_topic_index:get_topic(Match) || Match <- Matches] + ). t_match_fast_forward(Config) -> M = get_module(Config), From e5041de9ccf6f298b37aad1512f1227a702f05b9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 23 Aug 2023 09:58:57 -0300 Subject: [PATCH 054/119] fix(gcp_consumer): handle 401 errors Fixes https://emqx.atlassian.net/browse/EMQX-10852 --- ...emqx_bridge_gcp_pubsub_consumer_worker.erl | 16 ++- .../emqx_bridge_gcp_pubsub_impl_consumer.erl | 23 +++- .../emqx_bridge_gcp_pubsub_consumer_SUITE.erl | 130 ++++++++++++++++++ 3 files changed, 166 insertions(+), 3 deletions(-) diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_worker.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_worker.erl index d984b42ed..291ace7e0 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_worker.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_worker.erl @@ -218,6 +218,8 @@ handle_continue(?ensure_subscription, State0) -> not_found -> %% there's nothing much to do if the topic suddenly doesn't exist anymore. {stop, {error, topic_not_found}, State0}; + bad_credentials -> + {stop, {error, bad_credentials}, State0}; permission_denied -> {stop, {error, permission_denied}, State0} end; @@ -295,6 +297,7 @@ handle_info(Msg, State0) -> terminate({error, Reason}, State) when Reason =:= topic_not_found; + Reason =:= bad_credentials; Reason =:= permission_denied -> #{ @@ -335,7 +338,7 @@ ensure_pull_timer(State = #{pull_retry_interval := PullRetryInterval}) -> State#{pull_timer := emqx_utils:start_timer(PullRetryInterval, pull)}. -spec ensure_subscription_exists(state()) -> - continue | retry | not_found | permission_denied | already_exists. + continue | retry | not_found | permission_denied | bad_credentials | already_exists. ensure_subscription_exists(State) -> ?tp(gcp_pubsub_consumer_worker_create_subscription_enter, #{}), #{ @@ -384,6 +387,17 @@ ensure_subscription_exists(State) -> } ), permission_denied; + {error, #{status_code := 401}} -> + %% bad credentials + ?tp( + warning, + "gcp_pubsub_consumer_worker_bad_credentials", + #{ + instance_id => InstanceId, + topic => Topic + } + ), + bad_credentials; {ok, #{status_code := 200}} -> ?tp( debug, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl index 998a95a48..5c726ef9b 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl @@ -94,6 +94,8 @@ on_get_status(InstanceId, State) -> {disconnected, State, {unhealthy_target, ?TOPIC_MESSAGE}}; {error, permission_denied} -> {disconnected, State, {unhealthy_target, ?PERMISSION_MESSAGE}}; + {error, bad_credentials} -> + {disconnected, State, {unhealthy_target, ?PERMISSION_MESSAGE}}; ok -> #{client := Client} = State, check_workers(InstanceId, Client) @@ -103,7 +105,12 @@ on_get_status(InstanceId, State) -> %% Health check API (signalled by consumer worker) %%------------------------------------------------------------------------------------------------- --spec mark_as_unhealthy(resource_id(), topic_not_found | permission_denied) -> ok. +-spec mark_as_unhealthy( + resource_id(), + topic_not_found + | permission_denied + | bad_credentials +) -> ok. mark_as_unhealthy(InstanceId, Reason) -> optvar:set(?OPTVAR_UNHEALTHY(InstanceId), Reason), ok. @@ -114,7 +121,12 @@ clear_unhealthy(InstanceId) -> ?tp(gcp_pubsub_consumer_clear_unhealthy, #{}), ok. --spec check_if_unhealthy(resource_id()) -> ok | {error, topic_not_found | permission_denied}. +-spec check_if_unhealthy(resource_id()) -> + ok + | {error, + topic_not_found + | permission_denied + | bad_credentials}. check_if_unhealthy(InstanceId) -> case optvar:peek(?OPTVAR_UNHEALTHY(InstanceId)) of {ok, Reason} -> @@ -164,6 +176,11 @@ start_consumers(InstanceId, Client, Config) -> throw( {unhealthy_target, ?PERMISSION_MESSAGE} ); + {error, bad_credentials} -> + _ = emqx_bridge_gcp_pubsub_client:stop(InstanceId), + throw( + {unhealthy_target, ?PERMISSION_MESSAGE} + ); {error, _} -> %% connection might be down; we'll have to check topic existence during health %% check, or the workers will kill themselves when they realized there's no @@ -242,6 +259,8 @@ check_for_topic_existence(Topic, Client) -> {error, not_found}; {error, #{status_code := 403}} -> {error, permission_denied}; + {error, #{status_code := 401}} -> + {error, bad_credentials}; {error, Reason} -> ?tp(warning, "gcp_pubsub_consumer_check_topic_error", #{reason => Reason}), {error, Reason} diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl index 681e5fed7..8dc6cd7c4 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl @@ -818,6 +818,61 @@ permission_denied_response() -> ) }}. +unauthenticated_response() -> + Msg = << + "Request had invalid authentication credentials. Expected OAuth 2 access token," + " login cookie or other valid authentication credential. " + "See https://developers.google.com/identity/sign-in/web/devconsole-project." + >>, + {error, #{ + body => + #{ + <<"error">> => + #{ + <<"code">> => 401, + <<"details">> => + [ + #{ + <<"@type">> => + <<"type.googleapis.com/google.rpc.ErrorInfo">>, + <<"domain">> => <<"googleapis.com">>, + <<"metadata">> => + #{ + <<"email">> => + <<"test-516@emqx-cloud-pubsub.iam.gserviceaccount.com">>, + <<"method">> => + <<"google.pubsub.v1.Publisher.CreateTopic">>, + <<"service">> => + <<"pubsub.googleapis.com">> + }, + <<"reason">> => <<"ACCOUNT_STATE_INVALID">> + } + ], + <<"message">> => Msg, + + <<"status">> => <<"UNAUTHENTICATED">> + } + }, + headers => + [ + {<<"www-authenticate">>, <<"Bearer realm=\"https://accounts.google.com/\"">>}, + {<<"vary">>, <<"X-Origin">>}, + {<<"vary">>, <<"Referer">>}, + {<<"content-type">>, <<"application/json; charset=UTF-8">>}, + {<<"date">>, <<"Wed, 23 Aug 2023 12:41:40 GMT">>}, + {<<"server">>, <<"ESF">>}, + {<<"cache-control">>, <<"private">>}, + {<<"x-xss-protection">>, <<"0">>}, + {<<"x-frame-options">>, <<"SAMEORIGIN">>}, + {<<"x-content-type-options">>, <<"nosniff">>}, + {<<"alt-svc">>, <<"h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000">>}, + {<<"accept-ranges">>, <<"none">>}, + {<<"vary">>, <<"Origin,Accept-Encoding">>}, + {<<"transfer-encoding">>, <<"chunked">>} + ], + status_code => 401 + }}. + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -2125,6 +2180,81 @@ t_permission_denied_worker(Config) -> ), ok. +t_unauthenticated_topic_check(Config) -> + [#{pubsub_topic := PubSubTopic}] = ?config(topic_mapping, Config), + ResourceId = resource_id(Config), + ?check_trace( + begin + %% the emulator does not check any credentials + emqx_common_test_helpers:with_mock( + emqx_bridge_gcp_pubsub_client, + query_sync, + fun(PreparedRequest = {prepared_request, {Method, Path, _Body}}, Client) -> + RE = iolist_to_binary(["/topics/", PubSubTopic, "$"]), + case {Method =:= get, re:run(Path, RE)} of + {true, {match, _}} -> + unauthenticated_response(); + _ -> + meck:passthrough([PreparedRequest, Client]) + end + end, + fun() -> + {{ok, _}, {ok, _}} = + ?wait_async_action( + create_bridge(Config), + #{?snk_kind := gcp_pubsub_stop}, + 5_000 + ), + ?assertMatch( + {ok, disconnected}, + emqx_resource_manager:health_check(ResourceId) + ), + ?assertMatch( + {ok, _Group, #{error := {unhealthy_target, "Permission denied" ++ _}}}, + emqx_resource_manager:lookup_cached(ResourceId) + ), + ok + end + ), + ok + end, + [] + ), + ok. + +t_unauthenticated_worker(Config) -> + ?check_trace( + begin + emqx_common_test_helpers:with_mock( + emqx_bridge_gcp_pubsub_client, + query_sync, + fun(PreparedRequest = {prepared_request, {Method, _Path, _Body}}, Client) -> + case Method =:= put of + true -> + unauthenticated_response(); + false -> + meck:passthrough([PreparedRequest, Client]) + end + end, + fun() -> + {{ok, _}, {ok, _}} = + ?wait_async_action( + create_bridge( + Config + ), + #{?snk_kind := gcp_pubsub_consumer_worker_terminate}, + 10_000 + ), + + ok + end + ), + ok + end, + [] + ), + ok. + t_cluster_subscription(Config) -> [ #{ From 0037295c83eaff75ee14e9b8721f3ae6a6da21a1 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 23 Aug 2023 12:16:29 -0300 Subject: [PATCH 055/119] ci(test_docker): export enterprise image with elixir for testing --- .github/workflows/build_docker_for_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_docker_for_test.yaml b/.github/workflows/build_docker_for_test.yaml index 548d5e2cd..758cdd1cf 100644 --- a/.github/workflows/build_docker_for_test.yaml +++ b/.github/workflows/build_docker_for_test.yaml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest env: EMQX_NAME: ${{ matrix.profile }} - PKG_VSN: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }} + PKG_VSN: ${{ startsWith(matrix.profile, 'emqx-enterprise') && inputs.version-emqx-enterprise || inputs.version-emqx }} OTP_VSN: ${{ inputs.otp_vsn }} ELIXIR_VSN: ${{ inputs.elixir_vsn }} @@ -36,6 +36,7 @@ jobs: - emqx - emqx-enterprise - emqx-elixir + - emqx-enterprise-elixir steps: - uses: actions/checkout@v3 @@ -58,4 +59,3 @@ jobs: name: "${{ env.EMQX_NAME }}-docker" path: "${{ env.EMQX_NAME }}-docker-${{ env.PKG_VSN }}.tar.gz" retention-days: 3 - From 60e6217496a8e43d58adfa7e0f6c87727af9320c Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Wed, 23 Aug 2023 18:33:57 -0300 Subject: [PATCH 056/119] fix(kafka): fix result handling when sending message with invalid header Fixes https://emqx.atlassian.net/browse/EMQX-10846 --- .../src/emqx_bridge_kafka_impl_producer.erl | 107 +++++++++++++----- .../emqx_bridge_kafka_impl_producer_SUITE.erl | 84 ++++++++++++++ changes/ee/fix-11508.en.md | 1 + 3 files changed, 164 insertions(+), 28 deletions(-) create mode 100644 changes/ee/fix-11508.en.md diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index ea6666ea0..6ae7c93fa 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -213,7 +213,7 @@ on_stop(InstanceId, _State) -> ok. on_query( - _InstId, + InstId, {send_message, Message}, #{ message_template := Template, @@ -229,19 +229,34 @@ on_query( ext_headers_tokens => KafkaExtHeadersTokens, headers_val_encode_mode => KafkaHeadersValEncodeMode }, - KafkaMessage = render_message(Template, KafkaHeaders, Message), - ?tp( - emqx_bridge_kafka_impl_producer_sync_query, - #{headers_config => KafkaHeaders, instance_id => _InstId} - ), try - {_Partition, _Offset} = wolff:send_sync(Producers, [KafkaMessage], SyncTimeout), - ok + KafkaMessage = render_message(Template, KafkaHeaders, Message), + ?tp( + emqx_bridge_kafka_impl_producer_sync_query, + #{headers_config => KafkaHeaders, instance_id => InstId} + ), + do_send_msg(sync, KafkaMessage, Producers, SyncTimeout) catch - error:{producer_down, _} = Reason -> - {error, Reason}; - error:timeout -> - {error, timeout} + throw:{bad_kafka_header, _} = Error -> + ?tp( + emqx_bridge_kafka_impl_producer_sync_query_failed, + #{ + headers_config => KafkaHeaders, + instance_id => InstId, + reason => Error + } + ), + {error, {unrecoverable_error, Error}}; + throw:{bad_kafka_headers, _} = Error -> + ?tp( + emqx_bridge_kafka_impl_producer_sync_query_failed, + #{ + headers_config => KafkaHeaders, + instance_id => InstId, + reason => Error + } + ), + {error, {unrecoverable_error, Error}} end. %% @doc The callback API for rule-engine (or bridge without rules) @@ -251,7 +266,7 @@ on_query( %% E.g. the output of rule-engine process chain %% or the direct mapping from an MQTT message. on_query_async( - _InstId, + InstId, {send_message, Message}, AsyncReplyFn, #{ @@ -267,21 +282,35 @@ on_query_async( ext_headers_tokens => KafkaExtHeadersTokens, headers_val_encode_mode => KafkaHeadersValEncodeMode }, - KafkaMessage = render_message(Template, KafkaHeaders, Message), - ?tp( - emqx_bridge_kafka_impl_producer_async_query, - #{headers_config => KafkaHeaders, instance_id => _InstId} - ), - %% * Must be a batch because wolff:send and wolff:send_sync are batch APIs - %% * Must be a single element batch because wolff books calls, but not batch sizes - %% for counters and gauges. - Batch = [KafkaMessage], - %% The retuned information is discarded here. - %% If the producer process is down when sending, this function would - %% raise an error exception which is to be caught by the caller of this callback - {_Partition, Pid} = wolff:send(Producers, Batch, {fun ?MODULE:on_kafka_ack/3, [AsyncReplyFn]}), - %% this Pid is so far never used because Kafka producer is by-passing the buffer worker - {ok, Pid}. + try + KafkaMessage = render_message(Template, KafkaHeaders, Message), + ?tp( + emqx_bridge_kafka_impl_producer_async_query, + #{headers_config => KafkaHeaders, instance_id => InstId} + ), + do_send_msg(async, KafkaMessage, Producers, AsyncReplyFn) + catch + throw:{bad_kafka_header, _} = Error -> + ?tp( + emqx_bridge_kafka_impl_producer_async_query_failed, + #{ + headers_config => KafkaHeaders, + instance_id => InstId, + reason => Error + } + ), + {error, {unrecoverable_error, Error}}; + throw:{bad_kafka_headers, _} = Error -> + ?tp( + emqx_bridge_kafka_impl_producer_async_query_failed, + #{ + headers_config => KafkaHeaders, + instance_id => InstId, + reason => Error + } + ), + {error, {unrecoverable_error, Error}} + end. compile_message_template(T) -> KeyTemplate = maps:get(key, T, <<"${.clientid}">>), @@ -337,6 +366,28 @@ render_timestamp(Template, Message) -> erlang:system_time(millisecond) end. +do_send_msg(sync, KafkaMessage, Producers, SyncTimeout) -> + try + {_Partition, _Offset} = wolff:send_sync(Producers, [KafkaMessage], SyncTimeout), + ok + catch + error:{producer_down, _} = Reason -> + {error, Reason}; + error:timeout -> + {error, timeout} + end; +do_send_msg(async, KafkaMessage, Producers, AsyncReplyFn) -> + %% * Must be a batch because wolff:send and wolff:send_sync are batch APIs + %% * Must be a single element batch because wolff books calls, but not batch sizes + %% for counters and gauges. + Batch = [KafkaMessage], + %% The retuned information is discarded here. + %% If the producer process is down when sending, this function would + %% raise an error exception which is to be caught by the caller of this callback + {_Partition, Pid} = wolff:send(Producers, Batch, {fun ?MODULE:on_kafka_ack/3, [AsyncReplyFn]}), + %% this Pid is so far never used because Kafka producer is by-passing the buffer worker + {ok, Pid}. + %% Wolff producer never gives up retrying %% so there can only be 'ok' results. on_kafka_ack(_Partition, Offset, {ReplyFn, Args}) when is_integer(Offset) -> diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index d93b6dd7d..5e068dafd 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -677,6 +677,90 @@ t_wrong_headers(_Config) -> ), ok. +t_wrong_headers_from_message(Config) -> + HostsString = kafka_hosts_string_sasl(), + AuthSettings = valid_sasl_plain_settings(), + Hash = erlang:phash2([HostsString, ?FUNCTION_NAME]), + Type = ?BRIDGE_TYPE, + Name = "kafka_bridge_name_" ++ erlang:integer_to_list(Hash), + ResourceId = emqx_bridge_resource:resource_id(Type, Name), + BridgeId = emqx_bridge_resource:bridge_id(Type, Name), + KafkaTopic = "test-topic-one-partition", + Conf = config_with_headers(#{ + "authentication" => AuthSettings, + "kafka_hosts_string" => HostsString, + "kafka_topic" => KafkaTopic, + "instance_id" => ResourceId, + "kafka_headers" => <<"${payload}">>, + "producer" => #{ + "kafka" => #{ + "buffer" => #{ + "memory_overload_protection" => false + } + } + }, + "ssl" => #{} + }), + {ok, #{config := ConfigAtom1}} = emqx_bridge:create( + Type, erlang:list_to_atom(Name), Conf + ), + ConfigAtom = ConfigAtom1#{bridge_name => Name}, + {ok, State} = ?PRODUCER:on_start(ResourceId, ConfigAtom), + Time1 = erlang:unique_integer(), + Payload1 = <<"wrong_header">>, + Msg1 = #{ + clientid => integer_to_binary(Time1), + payload => Payload1, + timestamp => Time1 + }, + ?assertError( + {badmatch, {error, {unrecoverable_error, {bad_kafka_headers, Payload1}}}}, + send(Config, ResourceId, Msg1, State) + ), + Time2 = erlang:unique_integer(), + Payload2 = <<"[{\"foo\":\"bar\"}, {\"foo2\":\"bar2\"}]">>, + Msg2 = #{ + clientid => integer_to_binary(Time2), + payload => Payload2, + timestamp => Time2 + }, + ?assertError( + {badmatch, {error, {unrecoverable_error, {bad_kafka_header, [{<<"foo">>, <<"bar">>}]}}}}, + send(Config, ResourceId, Msg2, State) + ), + Time3 = erlang:unique_integer(), + Payload3 = <<"[{\"key\":\"foo\"}, {\"value\":\"bar\"}]">>, + Msg3 = #{ + clientid => integer_to_binary(Time3), + payload => Payload3, + timestamp => Time3 + }, + ?assertError( + {badmatch, {error, {unrecoverable_error, {bad_kafka_header, [{<<"key">>, <<"foo">>}]}}}}, + send(Config, ResourceId, Msg3, State) + ), + Time4 = erlang:unique_integer(), + Payload4 = <<"[{\"key\":\"foo\", \"value\":\"bar\"}]">>, + Msg4 = #{ + clientid => integer_to_binary(Time4), + payload => Payload4, + timestamp => Time4 + }, + ?assertError( + {badmatch, + {error, + {unrecoverable_error, + {bad_kafka_header, [{<<"key">>, <<"foo">>}, {<<"value">>, <<"bar">>}]}}}}, + send(Config, ResourceId, Msg4, State) + ), + %% TODO: refactor those into init/end per testcase + ok = ?PRODUCER:on_stop(ResourceId, State), + ?assertEqual([], supervisor:which_children(wolff_client_sup)), + ?assertEqual([], supervisor:which_children(wolff_producers_sup)), + ok = emqx_bridge_resource:remove(BridgeId), + delete_all_bridges(), + ok. + %%------------------------------------------------------------------------------ %% Helper functions %%------------------------------------------------------------------------------ diff --git a/changes/ee/fix-11508.en.md b/changes/ee/fix-11508.en.md new file mode 100644 index 000000000..54ea90db3 --- /dev/null +++ b/changes/ee/fix-11508.en.md @@ -0,0 +1 @@ +Fix message error handling on Kafka bridge when headers translate to an invalid value. From c215fe37366df90361db6eddc6faffe46fb991b9 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 24 Aug 2023 22:14:14 +0800 Subject: [PATCH 057/119] fix: stop otel deps appication before reboot --- apps/emqx_machine/src/emqx_machine_boot.erl | 2 + apps/emqx_opentelemetry/src/emqx_otel.erl | 24 ++++++++++- .../src/emqx_otel_config.erl | 6 +-- apps/emqx_opentelemetry/src/emqx_otel_sup.erl | 42 +++++-------------- 4 files changed, 38 insertions(+), 36 deletions(-) diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 82b909b4f..cb05280f5 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -62,6 +62,8 @@ stop_apps() -> ?SLOG(notice, #{msg => "stopping_emqx_apps"}), _ = emqx_alarm_handler:unload(), ok = emqx_conf_app:unset_config_loaded(), + %% Mute otel deps application. + _ = emqx_otel:stop_otel(), lists:foreach(fun stop_one_app/1, lists:reverse(sorted_reboot_apps())). %% Those port apps are terminated after the main apps diff --git a/apps/emqx_opentelemetry/src/emqx_otel.erl b/apps/emqx_opentelemetry/src/emqx_otel.erl index 408fd4c58..f7ded620b 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel.erl @@ -17,10 +17,25 @@ -module(emqx_otel). -include_lib("emqx/include/logger.hrl"). --export([start_link/1, cleanup/0]). +-export([start_otel/1, stop_otel/0]). -export([get_cluster_gauge/1, get_stats_gauge/1, get_vm_gauge/1, get_metric_counter/1]). +-export([start_link/1]). -export([init/1, handle_continue/2, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). +-define(SUPERVISOR, emqx_otel_sup). + +start_otel(Conf) -> + Spec = emqx_otel_sup:worker_spec(?MODULE, Conf), + assert_started(supervisor:start_child(?SUPERVISOR, Spec)). + +stop_otel() -> + ok = cleanup(), + case supervisor:terminate_child(?SUPERVISOR, ?MODULE) of + ok -> supervisor:delete_child(?SUPERVISOR, ?MODULE); + {error, not_found} -> ok; + Error -> Error + end. + start_link(Conf) -> gen_server:start_link({local, ?MODULE}, ?MODULE, Conf, []). @@ -47,7 +62,7 @@ setup(Conf = #{enable := true}) -> ensure_apps(Conf), create_metric_views(); setup(_Conf) -> - cleanup(), + ok = cleanup(), ok. ensure_apps(Conf) -> @@ -225,3 +240,8 @@ create_counter(Meter, Counters, CallBack) -> normalize_name(Name) -> list_to_existing_atom(lists:flatten(string:replace(atom_to_list(Name), "_", ".", all))). + +assert_started({ok, _Pid}) -> ok; +assert_started({ok, _Pid, _Info}) -> ok; +assert_started({error, {already_started, _Pid}}) -> ok; +assert_started({error, Reason}) -> {error, Reason}. diff --git a/apps/emqx_opentelemetry/src/emqx_otel_config.erl b/apps/emqx_opentelemetry/src/emqx_otel_config.erl index 3df535890..45a38e134 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_config.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_config.erl @@ -52,7 +52,7 @@ post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> ok. ensure_otel(#{enable := true} = Conf) -> - _ = emqx_otel_sup:stop_otel(), - emqx_otel_sup:start_otel(Conf); + _ = emqx_otel:stop_otel(), + emqx_otel:start_otel(Conf); ensure_otel(#{enable := false}) -> - emqx_otel_sup:stop_otel(). + emqx_otel:stop_otel(). diff --git a/apps/emqx_opentelemetry/src/emqx_otel_sup.erl b/apps/emqx_opentelemetry/src/emqx_otel_sup.erl index 6d04a0c2e..19f22cb84 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_sup.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_sup.erl @@ -19,36 +19,21 @@ -export([start_link/0]). -export([init/1]). --export([start_otel/1]). --export([stop_otel/0]). +-export([worker_spec/2]). --define(CHILD(Mod, Opts), #{ - id => Mod, - start => {Mod, start_link, [Opts]}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [Mod] -}). - --define(WORKER, emqx_otel). +worker_spec(Mod, Opts) -> + #{ + id => Mod, + start => {Mod, start_link, [Opts]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [Mod] + }. start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). --spec start_otel(map()) -> ok. -start_otel(Conf) -> - assert_started(supervisor:start_child(?MODULE, ?CHILD(?WORKER, Conf))). - --spec stop_otel() -> ok | {error, term()}. -stop_otel() -> - ok = emqx_otel:cleanup(), - case supervisor:terminate_child(?MODULE, ?WORKER) of - ok -> supervisor:delete_child(?MODULE, ?WORKER); - {error, not_found} -> ok; - Error -> Error - end. - init([]) -> SupFlags = #{ strategy => one_for_one, @@ -58,11 +43,6 @@ init([]) -> Children = case emqx_conf:get([opentelemetry]) of #{enable := false} -> []; - #{enable := true} = Conf -> [?CHILD(?WORKER, Conf)] + #{enable := true} = Conf -> [worker_spec(emqx_otel, Conf)] end, {ok, {SupFlags, Children}}. - -assert_started({ok, _Pid}) -> ok; -assert_started({ok, _Pid, _Info}) -> ok; -assert_started({error, {already_started, _Pid}}) -> ok; -assert_started({error, Reason}) -> {error, Reason}. From 0381ac0410bdcb41b139dfeb26cd66cc6cd5ce17 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 24 Aug 2023 10:10:50 -0300 Subject: [PATCH 058/119] fix(kafka_producer): use correct timestamp template field Fixes https://emqx.atlassian.net/browse/EMQX-10847 --- .../src/emqx_bridge_kafka_impl_producer.erl | 2 +- .../emqx_bridge_kafka_impl_producer_SUITE.erl | 46 ++++++++++++++++++- changes/ee/fix-11513.en.md | 1 + 3 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 changes/ee/fix-11513.en.md diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index ea6666ea0..d8c934e6b 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -286,7 +286,7 @@ on_query_async( compile_message_template(T) -> KeyTemplate = maps:get(key, T, <<"${.clientid}">>), ValueTemplate = maps:get(value, T, <<"${.}">>), - TimestampTemplate = maps:get(value, T, <<"${.timestamp}">>), + TimestampTemplate = maps:get(timestamp, T, <<"${.timestamp}">>), #{ key => preproc_tmpl(KeyTemplate), value => preproc_tmpl(ValueTemplate), diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index d93b6dd7d..b920b39ae 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -470,7 +470,51 @@ t_failed_creation_then_fix(Config) -> delete_all_bridges(), ok. -t_table_removed(_Config) -> +t_custom_timestamp(_Config) -> + HostsString = kafka_hosts_string_sasl(), + AuthSettings = valid_sasl_plain_settings(), + Hash = erlang:phash2([HostsString, ?FUNCTION_NAME]), + Type = ?BRIDGE_TYPE, + Name = "kafka_bridge_name_" ++ erlang:integer_to_list(Hash), + ResourceId = emqx_bridge_resource:resource_id(Type, Name), + KafkaTopic = "test-topic-one-partition", + MQTTTopic = <<"t/local/kafka">>, + emqx:subscribe(MQTTTopic), + Conf0 = config(#{ + "authentication" => AuthSettings, + "kafka_hosts_string" => HostsString, + "local_topic" => MQTTTopic, + "kafka_topic" => KafkaTopic, + "instance_id" => ResourceId, + "ssl" => #{} + }), + Conf = emqx_utils_maps:deep_put( + [<<"kafka">>, <<"message">>, <<"timestamp">>], + Conf0, + <<"123">> + ), + {ok, _} = emqx_bridge:create(Type, erlang:list_to_atom(Name), Conf), + {ok, Offset} = resolve_kafka_offset(kafka_hosts(), KafkaTopic, 0), + ct:pal("base offset before testing ~p", [Offset]), + Time = erlang:unique_integer(), + BinTime = integer_to_binary(Time), + Msg = #{ + clientid => BinTime, + payload => <<"payload">>, + timestamp => Time + }, + emqx:publish(emqx_message:make(MQTTTopic, emqx_utils_json:encode(Msg))), + {ok, {_, [KafkaMsg]}} = + ?retry( + _Interval = 500, + _NAttempts = 20, + {ok, {_, [_]}} = brod:fetch(kafka_hosts(), KafkaTopic, _Partition = 0, Offset) + ), + ?assertMatch(#kafka_message{ts = 123, ts_type = create}, KafkaMsg), + delete_all_bridges(), + ok. + +t_nonexistent_topic(_Config) -> HostsString = kafka_hosts_string_sasl(), AuthSettings = valid_sasl_plain_settings(), Hash = erlang:phash2([HostsString, ?FUNCTION_NAME]), diff --git a/changes/ee/fix-11513.en.md b/changes/ee/fix-11513.en.md new file mode 100644 index 000000000..51d953933 --- /dev/null +++ b/changes/ee/fix-11513.en.md @@ -0,0 +1 @@ +Fixed a bug which prevented the Kafka Producer bridge from using the correct template for the `timestamp` field. From 016ae0524f79278d039789f1e93a9749cef58c55 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 24 Aug 2023 11:14:02 -0300 Subject: [PATCH 059/119] fix(aeh_producer): remove timestamp template field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes https://emqx.atlassian.net/browse/EMQX-10847 Checking the whole Kafka message from AEH, it seems like the timestamp type is append, which means that it’s the broker who controls the timestamp, and the timestamp defined by the producer is ignored. Ref: https://kafka.apache.org/documentation/#brokerconfigs_log.message.timestamp.type Example message consumed from AEH: ``` %{ "headers" => %{}, "key" => "", "offset" => 4, "topic" => "test0", "ts" => 1692879703006, "ts_type" => "append", "value" => "{\"username\":\"undefined\",\"topic\":\"t/aeh/produ\",\"timestamp\":1692879692189,\"qos\":0,\"publish_received_at\":1692879692189,\"pub_props\":{\"User-Property\":{}},\"peerhost\":\"undefined\",\"payload\":\"aaaa\",\"node\":\"emqx@127.0.0.1\",\"metadata\":{\"rule_id\":\"rule_aehp\"},\"id\":\"000603AA44B34E08F4AF000006E30003\",\"flags\":{},\"event\":\"message.publish\",\"clientid\":\"undefined\"}" } ``` Note the ts_type above is append. Example message from a Kafka broker whose ts type is create: ``` %{ "headers" => %{}, "key" => "", "offset" => 4, "topic" => "test-topic-three-partitions", "ts" => 1692881883668, "ts_type" => "create", "value" => "{\"username\":\"undefined\",\"topic\":\"t/kafka/produ\",\"timestamp\":1692881883668,\"qos\":0,\"publish_received_at\":1692881883668,\"pub_props\":{\"User-Property\":{}},\"peerhost\":\"undefined\",\"payload\":\"aaaaaa\",\"node\":\"emqx@127.0.0.1\",\"id\":\"000603AAC7529FEEF4AC000007050000\",\"flags\":{},\"event\":\"message.publish\",\"clientid\":\"undefined\"}" } ``` Unfortunately, I couldn’t find anywhere in AEH where that configuration could be changed. --- .../src/emqx_bridge_azure_event_hub.app.src | 2 +- .../src/emqx_bridge_azure_event_hub.erl | 6 ++++++ .../test/emqx_bridge_azure_event_hub_producer_SUITE.erl | 5 ----- .../test/emqx_bridge_azure_event_hub_tests.erl | 1 - 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src index e29e9c83a..43033b657 100644 --- a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_azure_event_hub, [ {description, "EMQX Enterprise Azure Event Hub Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl index 2d6343b74..c563a35d8 100644 --- a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl @@ -64,6 +64,10 @@ fields(producer_kafka_opts) -> kafka_producer_overrides() ), override_documentations(Fields); +fields(kafka_message) -> + Fields0 = emqx_bridge_kafka:fields(kafka_message), + Fields = proplists:delete(timestamp, Fields0), + override_documentations(Fields); fields(Method) -> Fields = emqx_bridge_kafka:fields(Method), override_documentations(Fields). @@ -85,6 +89,7 @@ desc(Name) -> struct_names() -> [ auth_username_password, + kafka_message, producer_kafka_opts ]. @@ -245,6 +250,7 @@ kafka_producer_overrides() -> default => no_compression, importance => ?IMPORTANCE_HIDDEN }), + message => mk(ref(kafka_message), #{}), required_acks => mk(enum([all_isr, leader_only]), #{default => all_isr}) }. diff --git a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_producer_SUITE.erl b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_producer_SUITE.erl index af4b87718..e77d724d0 100644 --- a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_producer_SUITE.erl +++ b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_producer_SUITE.erl @@ -65,10 +65,6 @@ init_per_suite(Config) -> end. end_per_suite(Config) -> - %% emqx_mgmt_api_test_util:end_suite(), - %% ok = emqx_common_test_helpers:stop_apps([emqx_conf]), - %% ok = emqx_connector_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_rule_engine]), - %% _ = application:stop(emqx_connector), Apps = ?config(tc_apps, Config), emqx_cth_suite:stop(Apps), ok. @@ -145,7 +141,6 @@ bridge_config(TestCase, Config) -> <<"message">> => #{ <<"key">> => <<"${.clientid}">>, - <<"timestamp">> => <<"${.timestamp}">>, <<"value">> => <<"${.}">> }, <<"partition_count_refresh_interval">> => <<"60s">>, diff --git a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_tests.erl b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_tests.erl index d624421c6..92d268d20 100644 --- a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_tests.erl +++ b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_tests.erl @@ -33,7 +33,6 @@ bridges.azure_event_hub_producer.my_producer { max_inflight = 10 message { key = \"${.clientid}\" - timestamp = \"${.timestamp}\" value = \"${.}\" } partition_count_refresh_interval = 60s From 535c7f8b4338afe74d558dcf4ffb78cee9e06bb2 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Thu, 24 Aug 2023 12:55:13 -0300 Subject: [PATCH 060/119] fix(kafka): avoid ClientId collision between Kafka and Azure bridges Fixes https://emqx.atlassian.net/browse/EMQX-10860 --- ..._bridge_azure_event_hub_producer_SUITE.erl | 36 +++++++++++++++++++ .../src/emqx_bridge_kafka_impl.erl | 11 +++--- .../src/emqx_bridge_kafka_impl_consumer.erl | 12 +++---- .../src/emqx_bridge_kafka_impl_producer.erl | 2 +- .../emqx_bridge_kafka_impl_consumer_SUITE.erl | 4 +-- 5 files changed, 50 insertions(+), 15 deletions(-) diff --git a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_producer_SUITE.erl b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_producer_SUITE.erl index af4b87718..65b0d54aa 100644 --- a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_producer_SUITE.erl +++ b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_producer_SUITE.erl @@ -12,6 +12,7 @@ -define(BRIDGE_TYPE, azure_event_hub_producer). -define(BRIDGE_TYPE_BIN, <<"azure_event_hub_producer">>). +-define(KAFKA_BRIDGE_TYPE, kafka). -define(APPS, [emqx_resource, emqx_bridge, emqx_rule_engine]). -import(emqx_common_test_helpers, [on_exit/1]). @@ -281,3 +282,38 @@ t_sync_query(Config) -> emqx_bridge_kafka_impl_producer_sync_query ), ok. + +t_same_name_azure_kafka_bridges(AehConfig) -> + ConfigKafka = lists:keyreplace(bridge_type, 1, AehConfig, {bridge_type, ?KAFKA_BRIDGE_TYPE}), + BridgeName = ?config(bridge_name, AehConfig), + AehResourceId = emqx_bridge_testlib:resource_id(AehConfig), + TracePoint = emqx_bridge_kafka_impl_producer_sync_query, + %% creates the AEH bridge and check it's working + ok = emqx_bridge_testlib:t_sync_query( + AehConfig, + fun make_message/0, + fun(Res) -> ?assertEqual(ok, Res) end, + TracePoint + ), + %% than creates a Kafka bridge with same name and delete it after creation + ok = emqx_bridge_testlib:t_create_via_http(ConfigKafka), + ?assertMatch( + {{ok, _}, {ok, _}}, + ?wait_async_action( + emqx_bridge:disable_enable(disable, ?KAFKA_BRIDGE_TYPE, BridgeName), + #{?snk_kind := kafka_producer_stopped}, + 5_000 + ) + ), + % check that AEH bridge is still working + ?check_trace( + begin + Message = {send_message, make_message()}, + ?assertEqual(ok, emqx_resource:simple_sync_query(AehResourceId, Message)), + ok + end, + fun(Trace) -> + ?assertMatch([#{instance_id := AehResourceId}], ?of_kind(TracePoint, Trace)) + end + ), + ok. diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl index bbdb4f3c7..64ac8cf99 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl @@ -7,7 +7,7 @@ -export([ hosts/1, - make_client_id/2, + make_client_id/1, sasl/1, socket_opts/1 ]). @@ -24,10 +24,11 @@ hosts(Hosts) when is_list(Hosts) -> kpro:parse_endpoints(Hosts). %% Client ID is better to be unique to make it easier for Kafka side trouble shooting. -make_client_id(KafkaType0, BridgeName0) -> - KafkaType = to_bin(KafkaType0), - BridgeName = to_bin(BridgeName0), - iolist_to_binary([KafkaType, ":", BridgeName, ":", atom_to_list(node())]). +make_client_id(InstanceId) -> + InstanceIdBin0 = to_bin(InstanceId), + % Removing the <<"bridge:">> from beginning for backward compatibility + InstanceIdBin = binary:replace(InstanceIdBin0, <<"bridge:">>, <<>>), + iolist_to_binary([InstanceIdBin, ":", atom_to_list(node())]). sasl(none) -> undefined; diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl index b8abb928c..06176ac93 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl @@ -121,7 +121,6 @@ on_start(ResourceId, Config) -> #{ authentication := Auth, bootstrap_hosts := BootstrapHosts0, - bridge_name := BridgeName, hookpoint := _, kafka := #{ max_batch_bytes := _, @@ -134,9 +133,8 @@ on_start(ResourceId, Config) -> topic_mapping := _ } = Config, BootstrapHosts = emqx_bridge_kafka_impl:hosts(BootstrapHosts0), - KafkaType = kafka_consumer, %% Note: this is distinct per node. - ClientID = make_client_id(ResourceId, KafkaType, BridgeName), + ClientID = make_client_id(ResourceId), ClientOpts0 = case Auth of none -> []; @@ -517,11 +515,11 @@ is_dry_run(ResourceId) -> string:equal(TestIdStart, ResourceId) end. --spec make_client_id(resource_id(), kafka_consumer, atom() | binary()) -> atom(). -make_client_id(ResourceId, KafkaType, KafkaName) -> - case is_dry_run(ResourceId) of +-spec make_client_id(resource_id()) -> atom(). +make_client_id(InstanceId) -> + case is_dry_run(InstanceId) of false -> - ClientID0 = emqx_bridge_kafka_impl:make_client_id(KafkaType, KafkaName), + ClientID0 = emqx_bridge_kafka_impl:make_client_id(InstanceId), binary_to_atom(ClientID0); true -> %% It is a dry run and we don't want to leak too many diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index ea6666ea0..c25c849e5 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -65,7 +65,7 @@ on_start(InstId, Config) -> ok = emqx_resource:allocate_resource(InstId, ?kafka_resource_id, ResourceId), _ = maybe_install_wolff_telemetry_handlers(ResourceId), Hosts = emqx_bridge_kafka_impl:hosts(Hosts0), - ClientId = emqx_bridge_kafka_impl:make_client_id(BridgeType, BridgeName), + ClientId = emqx_bridge_kafka_impl:make_client_id(InstId), ok = emqx_resource:allocate_resource(InstId, ?kafka_client_id, ClientId), ClientConfig = #{ min_metadata_refresh_interval => MinMetaRefreshInterval, diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 2d8355e8e..756fa8b3b 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -898,8 +898,8 @@ ensure_connected(Config) -> ok. consumer_clientid(Config) -> - KafkaName = ?config(kafka_name, Config), - binary_to_atom(emqx_bridge_kafka_impl:make_client_id(kafka_consumer, KafkaName)). + ResourceId = resource_id(Config), + binary_to_atom(emqx_bridge_kafka_impl:make_client_id(ResourceId)). get_client_connection(Config) -> KafkaHost = ?config(kafka_host, Config), From cf45e80c71c008e604a5f95072015d80fae86eeb Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 25 Aug 2023 01:47:11 +0400 Subject: [PATCH 061/119] feat(topicidx): iterate on trie search implementation This improves matching performance and decreases GC pressure on synthetic workloads. --- apps/emqx/src/emqx_trie_search.erl | 247 +++++++++++++--------- apps/emqx/test/emqx_topic_index_SUITE.erl | 13 ++ rebar.config.erl | 3 +- 3 files changed, 158 insertions(+), 105 deletions(-) diff --git a/apps/emqx/src/emqx_trie_search.erl b/apps/emqx/src/emqx_trie_search.erl index 71bc8d8a0..95174f292 100644 --- a/apps/emqx/src/emqx_trie_search.erl +++ b/apps/emqx/src/emqx_trie_search.erl @@ -110,16 +110,6 @@ -type nextf() :: fun((key(_) | base_key()) -> ?END | key(_)). -type opts() :: [unique | return_first]. -%% Holds the constant values of each search. --record(ctx, { - %% A function which can quickly find the immediate-next record of the given prefix - nextf :: nextf(), - %% The initial words of a topic - words0 :: [word()], - %% Return as soon as there is one match found - return_first :: boolean() -}). - %% @doc Make a search-key for the given topic. -spec make_key(emqx_types:topic(), ID) -> key(ID). make_key(Topic, ID) when is_binary(Topic) -> @@ -147,29 +137,29 @@ get_topic({Filter, _ID}) when is_list(Filter) -> get_topic({Topic, _ID}) -> Topic. +-compile({inline, [base/1, move_up/2, match_add/2, compare/3]}). + %% Make the base-key which can be used to locate the desired search target. base(Prefix) -> {Prefix, {}}. -%% Move the search target to the key next to the given Base. -move_up(#ctx{nextf = NextF}, Base) -> - NextF(Base). +base_init([W = <<"$", _/bytes>> | _]) -> + base([W]); +base_init(_) -> + base([]). -%% Add the given key to the accumulation. -add(#ctx{return_first = true}, _Acc, Key) -> - throw({return_first, Key}); -add(_C, Acc, Key) -> - match_add(Key, Acc). +%% Move the search target to the key next to the given Base. +move_up(NextF, Base) -> + NextF(Base). %% @doc Match given topic against the index and return the first match, or `false` if %% no match is found. -spec match(emqx_types:topic(), nextf()) -> false | key(_). match(Topic, NextF) -> try search(Topic, NextF, [return_first]) of - [] -> - false + _ -> false catch - throw:{return_first, Res} -> + throw:{first, Res} -> Res end. @@ -182,110 +172,159 @@ matches(Topic, NextF, Opts) -> %% @doc Entrypoint of the search for a given topic. search(Topic, NextF, Opts) -> Words = words(Topic), - Context = #ctx{ - nextf = NextF, - words0 = Words, - return_first = proplists:get_bool(return_first, Opts) - }, + Base = base_init(Words), + ORetFirst = proplists:get_bool(return_first, Opts), + OUnique = proplists:get_bool(unique, Opts), Acc0 = - case proplists:get_bool(unique, Opts) of + case ORetFirst of true -> + first; + false when OUnique -> #{}; false -> [] end, - Base = - case hd(Words) of - <<$$, _/binary>> -> - %% skip all filters starts with # or + - base([hd(Words)]); - _ -> - base([]) + Matches = + case search_new(Words, Base, NextF, Acc0) of + {Cursor, Acc} -> + match_topics(Topic, Cursor, NextF, Acc); + Acc -> + Acc end, - {MaybeEnd, Acc1} = search_new(Context, Base, Acc0), - Acc = match_topics(Context, Topic, MaybeEnd, Acc1), - case is_map(Acc) of + case is_map(Matches) of true -> - maps:values(Acc); + maps:values(Matches); false -> - Acc + Matches end. %% The recursive entrypoint of the trie-search algorithm. %% Always start from the initial prefix and words. -search_new(#ctx{words0 = Words} = C, NewBase, Acc) -> - case move_up(C, NewBase) of +search_new(Words0, NewBase, NextF, Acc) -> + case move_up(NextF, NewBase) of ?END -> - {?END, Acc}; - {Filter, _} = T -> - search_plus(C, Words, Filter, [], T, Acc) + Acc; + Cursor -> + search_up(Words0, Cursor, NextF, Acc) end. -%% Try to use '+' as the next word in the prefix. -search_plus(C, [W, X | Words], [W, X | Filter], RPrefix, T, Acc) -> - %% Directly append the current word to the matching prefix (RPrefix). - %% Micro optimization: try not to call the next clause because - %% it is not a continuation. - search_plus(C, [X | Words], [X | Filter], [W | RPrefix], T, Acc); -search_plus(C, [W | Words], ['+' | _] = Filter, RPrefix, T, Acc) -> - case search_up(C, '+', Words, Filter, RPrefix, T, Acc) of - {T, Acc1} -> - search_up(C, W, Words, Filter, RPrefix, T, Acc1); - TargetMoved -> - TargetMoved - end; -search_plus(C, [W | Words], Filter, RPrefix, T, Acc) -> - %% not a plus - search_up(C, W, Words, Filter, RPrefix, T, Acc). - %% Search to the bigger end of ordered collection of topics and topic-filters. -search_up(C, Word, Words, Filter, RPrefix, T, Acc) -> - case compare(Word, Filter, Words) of - {match, full} -> - search_new(C, T, add(C, Acc, T)); - {match, prefix} -> - search_new(C, T, Acc); +search_up(Words, {Filter, _} = Cursor, NextF, Acc) -> + case compare(Filter, Words, false) of + match_full -> + search_new(Words, Cursor, NextF, match_add(Cursor, Acc)); + match_prefix -> + search_new(Words, Cursor, NextF, Acc); lower -> - {T, Acc}; - higher -> - NewBase = base(lists:reverse([Word | RPrefix])), - search_new(C, NewBase, Acc); - shorter -> - search_plus(C, Words, tl(Filter), [Word | RPrefix], T, Acc) + {Cursor, Acc}; + [SeekWord | FilterTail] -> + % NOTE + % This is a seek instruction. + % If we visualize the `Filter` as `FilterHead ++ [_] ++ FilterTail`, we need to + % seek to `FilterHead ++ [SeekWord]`. It carries the `FilterTail` because it's + % much cheaper to return it from `compare/3` than anything more usable. + NewBase = base(seek(SeekWord, Filter, FilterTail)), + search_new(Words, NewBase, NextF, Acc) end. -%% Compare prefix word then the next words in suffix against the search-target -%% topic or topic-filter. -compare(_, NotFilter, _) when is_binary(NotFilter) -> - lower; -compare(H, [H | Filter], Words) -> - compare(Filter, Words); -compare(_, ['#'], _Words) -> - {match, full}; -compare(H1, [H2 | _T2], _Words) when H1 < H2 -> - lower; -compare(_H, [_ | _], _Words) -> - higher. +seek(SeekWord, [_ | FilterTail], FilterTail) -> + [SeekWord]; +seek(SeekWord, [FilterWord | Rest], FilterTail) -> + [FilterWord | seek(SeekWord, Rest, FilterTail)]. -%% Now compare the filter suffix and the topic suffix. -compare([], []) -> - {match, full}; -compare([], _Words) -> - {match, prefix}; -compare(['#'], _Words) -> - {match, full}; -compare([_ | _], []) -> +compare(NotFilter, _, _) when is_binary(NotFilter) -> lower; -compare([_ | _], _Words) -> - %% cannot know if it's a match, lower, or higher, - %% must search with a longer prefix. - shorter. +compare([], [], _) -> + % NOTE + % Topic: a/b/c/d + % Filter: a/+/+/d + % We matched the topic to a topic filter exactly (possibly with pluses). + % We include it in the result set, and now need to try next entry in the table. + % Closest possible next entries that we must not miss: + % * a/+/+/d (same topic but a different ID) + % * a/+/+/d/# (also a match) + match_full; +compare([], _Words, _) -> + % NOTE + % Topic: a/b/c/d + % Filter: a/+/c + % We found out that a topic filter is a prefix of the topic (possibly with pluses). + % We discard it, and now need to try next entry in the table. + % Closest possible next entries that we must not miss: + % * a/+/c/# (which is a match) + % * a/+/c/+ (also a match) + % + % TODO + % We might probably instead seek to a/+/c/# right away. + match_prefix; +compare(['#'], _Words, _) -> + % NOTE + % Topic: a/b/c/d + % Filter: a/+/+/d/# + % We matched the topic to a topic filter with wildcard (possibly with pluses). + % We include it in the result set, and now need to try next entry in the table. + % Closest possible next entries that we must not miss: + % * a/+/+/d/# (same topic but a different ID) + match_full; +compare(['+' | TF], [HW | TW], _PrevBacktrack) -> + % NOTE + % We need to keep backtrack point each time we encounter a plus. To safely skip over + % parts of the search space, we may need last backtrack point when recursion terminates. + % See next clauses for examples. + compare(TF, TW, [HW | TF]); +compare([HW | TF], [HW | TW], Backtrack) -> + % NOTE + % Skip over the same word in both topic and filter, keeping the last backtrack point. + compare(TF, TW, Backtrack); +compare([HF | _], [HW | _], false) when HF > HW -> + % NOTE + % Topic: a/b/c/d + % Filter: a/b/c/e/1 + % The topic is lower than a topic filter. There's nowhere to backtrackto, we're out of + % the search space. We should stop the search. + lower; +compare([HF | _], [HW | _], Backtrack) when HF > HW -> + % NOTE + % Topic: a/b/c/d + % Filter: a/+/+/e/1 + % The topic is lower than a topic filter. There was a plus, last time at the 3rd level, + % we have a backtrack point to seek to: + % Seek: [c | e/1] + % We need to skip over part of search space, and seek to the next possible match: + % Next: a/+/c + Backtrack; +compare([_ | _], [], false) -> + % NOTE + % Topic: a/b/c/d + % Filter: a/b/c/d/1 + % The topic is lower than a topic filter. (since it's shorter). There's nowhere to + % backtrack to, we're out of the search space. We should stop the search. + lower; +compare([_ | _], [], Backtrack) -> + % NOTE + % Topic: a/b/c/d + % Filter: a/+/c/d/1 + % The topic is lower than a topic filter. There was a plus, last and only time at the + % 3rd level, we have a backtrack point: + % Seek: [b | c/d/1] + % Next: a/b + Backtrack; +compare([_HF | TF], [HW | _], _) -> + % NOTE + % Topic: a/b/c/d + % Filter: a/+/+/0/1/2 + % Topic is higher than the filter, we need to skip over to the next possible filter. + % Seek: [d | 0/1/2] + % Next: a/+/+/d + [HW | TF]. match_add(K = {_Filter, ID}, Acc = #{}) -> % NOTE: ensuring uniqueness by record ID Acc#{ID => K}; -match_add(K, Acc) -> - [K | Acc]. +match_add(K, Acc) when is_list(Acc) -> + [K | Acc]; +match_add(K, first) -> + throw({first, K}). -spec words(emqx_types:topic()) -> [word()]. words(Topic) when is_binary(Topic) -> @@ -301,12 +340,12 @@ word(<<"#">>) -> '#'; word(Bin) -> Bin. %% match non-wildcard topics -match_topics(#ctx{nextf = NextF} = C, Topic, {Topic, _} = Key, Acc) -> +match_topics(Topic, {Topic, _} = Key, NextF, Acc) -> %% found a topic match - match_topics(C, Topic, NextF(Key), add(C, Acc, Key)); -match_topics(#ctx{nextf = NextF} = C, Topic, {F, _}, Acc) when F < Topic -> + match_topics(Topic, NextF(Key), NextF, match_add(Key, Acc)); +match_topics(Topic, {F, _}, NextF, Acc) when F < Topic -> %% the last key is a filter, try jump to the topic - match_topics(C, Topic, NextF(base(Topic)), Acc); -match_topics(_C, _Topic, _Key, Acc) -> + match_topics(Topic, NextF(base(Topic)), NextF, Acc); +match_topics(_Topic, _Key, _NextF, Acc) -> %% gone pass the topic Acc. diff --git a/apps/emqx/test/emqx_topic_index_SUITE.erl b/apps/emqx/test/emqx_topic_index_SUITE.erl index 551cc2438..08056a16f 100644 --- a/apps/emqx/test/emqx_topic_index_SUITE.erl +++ b/apps/emqx/test/emqx_topic_index_SUITE.erl @@ -256,6 +256,19 @@ t_match_wildcard_edge_cases(Config) -> end, lists:foreach(F, Datasets). +t_prop_edgecase(Config) -> + M = get_module(Config), + Tab = M:new(), + Topic = <<"01/01">>, + Filters = [ + {1, <<>>}, + {2, <<"+/01">>}, + {3, <<>>}, + {4, <<"+/+/01">>} + ], + _ = [M:insert(F, N, <<>>, Tab) || {N, F} <- Filters], + ?assertMatch([2], [id(X) || X <- matches(M, Topic, Tab, [unique])]). + t_prop_matches(Config) -> M = get_module(Config), ?assert( diff --git a/rebar.config.erl b/rebar.config.erl index 02e946e15..8f26d11d8 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -190,7 +190,8 @@ test_deps() -> {meck, "0.9.2"}, {proper, "1.4.0"}, {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.5"}}}, - {erl_csv, "0.2.0"} + {erl_csv, "0.2.0"}, + {eministat, "0.10.1"} ]. common_compile_opts() -> From 8ab6eb56384267cbe57d6e7834102bab41e5b3ca Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 25 Aug 2023 05:54:40 +0800 Subject: [PATCH 062/119] test: start emqx_opentelemetry in emqx_machine SUITE --- apps/emqx_machine/test/emqx_machine_SUITE.erl | 7 ++++--- apps/emqx_opentelemetry/src/emqx_otel.erl | 13 +++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/apps/emqx_machine/test/emqx_machine_SUITE.erl b/apps/emqx_machine/test/emqx_machine_SUITE.erl index bd18c67aa..224732d1f 100644 --- a/apps/emqx_machine/test/emqx_machine_SUITE.erl +++ b/apps/emqx_machine/test/emqx_machine_SUITE.erl @@ -42,7 +42,7 @@ init_per_suite(Config) -> %% Unload emqx_authz to avoid reboot this application %% application:unload(emqx_authz), - emqx_common_test_helpers:start_apps([emqx_conf]), + emqx_common_test_helpers:start_apps([emqx_conf, emqx_opentelemetry]), application:set_env(emqx_machine, applications, [ emqx_prometheus, emqx_modules, @@ -56,12 +56,13 @@ init_per_suite(Config) -> emqx_exhook, emqx_authn, emqx_authz, - emqx_plugin + emqx_plugin, + emqx_opentelemetry ]), Config. end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([emqx_opentelemetry, emqx_conf]). init_per_testcase(t_custom_shard_transports, Config) -> OldConfig = application:get_env(emqx_machine, custom_shard_transports), diff --git a/apps/emqx_opentelemetry/src/emqx_otel.erl b/apps/emqx_opentelemetry/src/emqx_otel.erl index f7ded620b..141c8ff04 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel.erl @@ -30,10 +30,15 @@ start_otel(Conf) -> stop_otel() -> ok = cleanup(), - case supervisor:terminate_child(?SUPERVISOR, ?MODULE) of - ok -> supervisor:delete_child(?SUPERVISOR, ?MODULE); - {error, not_found} -> ok; - Error -> Error + case erlang:whereis(?SUPERVISOR) of + undefined -> + ok; + Pid -> + case supervisor:terminate_child(Pid, ?MODULE) of + ok -> supervisor:delete_child(Pid, ?MODULE); + {error, not_found} -> ok; + Error -> Error + end end. start_link(Conf) -> From fc1738188e0203bd9fc46b9d39a595d191abc9c0 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 25 Aug 2023 15:08:49 +0800 Subject: [PATCH 063/119] fix: packets_connack_sent is not incremented if the ack_flag field in the CONNACK packet is non-zero --- apps/emqx/src/emqx_channel.erl | 1 + apps/emqx/src/emqx_cm.erl | 10 ---------- apps/emqx/src/emqx_connection.erl | 1 - apps/emqx/src/emqx_metrics.erl | 2 +- apps/emqx/src/emqx_ws_connection.erl | 1 - apps/emqx/test/emqx_metrics_SUITE.erl | 15 +++++++++++++-- changes/ce/fix-11520.en.md | 1 + 7 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 changes/ce/fix-11520.en.md diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 87680358f..bf1729143 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -2218,6 +2218,7 @@ disconnect_and_shutdown(Reason, Reply, Channel) -> NChannel = ensure_disconnected(Reason, Channel), shutdown(Reason, Reply, NChannel). +-compile({inline, [sp/1, flag/1]}). sp(true) -> 1; sp(false) -> 0. diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index c680560fb..a78ba752e 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -35,8 +35,6 @@ insert_channel_info/3 ]). --export([connection_closed/1]). - -export([ get_chan_info/1, get_chan_info/2, @@ -192,14 +190,6 @@ do_unregister_channel({_ClientId, ChanPid} = Chan) -> ok = emqx_hooks:run('channel.unregistered', [ChanPid]), true. --spec connection_closed(emqx_types:clientid()) -> true. -connection_closed(ClientId) -> - connection_closed(ClientId, self()). - --spec connection_closed(emqx_types:clientid(), chan_pid()) -> true. -connection_closed(ClientId, ChanPid) -> - ets:delete_object(?CHAN_CONN_TAB, {ClientId, ChanPid}). - %% @doc Get info of a channel. -spec get_chan_info(emqx_types:clientid()) -> maybe(emqx_types:infos()). get_chan_info(ClientId) -> diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 70eb0d1e4..60f38e2bd 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -636,7 +636,6 @@ handle_msg( handle_msg({event, disconnected}, State = #state{channel = Channel}) -> ClientId = emqx_channel:info(clientid, Channel), emqx_cm:set_chan_info(ClientId, info(State)), - emqx_cm:connection_closed(ClientId), {ok, State}; handle_msg({event, _Other}, State = #state{channel = Channel}) -> ClientId = emqx_channel:info(clientid, Channel), diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index 5f5deadd3..ee86457b0 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -493,7 +493,7 @@ inc_sent(Packet) -> inc('packets.sent'), do_inc_sent(Packet). -do_inc_sent(?CONNACK_PACKET(ReasonCode)) -> +do_inc_sent(?CONNACK_PACKET(ReasonCode, _SessPresent)) -> (ReasonCode == ?RC_SUCCESS) orelse inc('packets.connack.error'), ((ReasonCode == ?RC_NOT_AUTHORIZED) orelse (ReasonCode == ?CONNACK_AUTH)) andalso diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index 93c29bf4f..37ce72d74 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -531,7 +531,6 @@ handle_info({event, connected}, State = #state{channel = Channel}) -> handle_info({event, disconnected}, State = #state{channel = Channel}) -> ClientId = emqx_channel:info(clientid, Channel), emqx_cm:set_chan_info(ClientId, info(State)), - emqx_cm:connection_closed(ClientId), return(State); handle_info({event, _Other}, State = #state{channel = Channel}) -> ClientId = emqx_channel:info(clientid, Channel), diff --git a/apps/emqx/test/emqx_metrics_SUITE.erl b/apps/emqx/test/emqx_metrics_SUITE.erl index 45a14a6f7..c9c384b44 100644 --- a/apps/emqx/test/emqx_metrics_SUITE.erl +++ b/apps/emqx/test/emqx_metrics_SUITE.erl @@ -122,6 +122,17 @@ t_inc_sent(_) -> with_metrics_server( fun() -> ok = emqx_metrics:inc_sent(?CONNACK_PACKET(0)), + ok = emqx_metrics:inc_sent(?CONNACK_PACKET(0, 1)), + ok = emqx_metrics:inc_sent( + ?CONNACK_PACKET(0, 1, #{ + 'Maximum-Packet-Size' => 1048576, + 'Retain-Available' => 1, + 'Shared-Subscription-Available' => 1, + 'Subscription-Identifier-Available' => 1, + 'Topic-Alias-Maximum' => 65535, + 'Wildcard-Subscription-Available' => 1 + }) + ), ok = emqx_metrics:inc_sent(?PUBLISH_PACKET(0, 0)), ok = emqx_metrics:inc_sent(?PUBLISH_PACKET(1, 0)), ok = emqx_metrics:inc_sent(?PUBLISH_PACKET(2, 0)), @@ -134,8 +145,8 @@ t_inc_sent(_) -> ok = emqx_metrics:inc_sent(?PACKET(?PINGRESP)), ok = emqx_metrics:inc_sent(?PACKET(?DISCONNECT)), ok = emqx_metrics:inc_sent(?PACKET(?AUTH)), - ?assertEqual(13, emqx_metrics:val('packets.sent')), - ?assertEqual(1, emqx_metrics:val('packets.connack.sent')), + ?assertEqual(15, emqx_metrics:val('packets.sent')), + ?assertEqual(3, emqx_metrics:val('packets.connack.sent')), ?assertEqual(3, emqx_metrics:val('messages.sent')), ?assertEqual(1, emqx_metrics:val('messages.qos0.sent')), ?assertEqual(1, emqx_metrics:val('messages.qos1.sent')), diff --git a/changes/ce/fix-11520.en.md b/changes/ce/fix-11520.en.md new file mode 100644 index 000000000..42675c5f5 --- /dev/null +++ b/changes/ce/fix-11520.en.md @@ -0,0 +1 @@ +Fixed issue where packets_connack_sent metric was not incremented on CONNACK packets sent with non-zero ack_flag From c8af5a702c68d3e848ca378e1babdeefcdaa10fe Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 25 Aug 2023 15:53:35 +0800 Subject: [PATCH 064/119] test: remove connection_closed from test --- apps/emqx/test/emqx_connection_SUITE.erl | 1 - apps/emqx/test/emqx_ws_connection_SUITE.erl | 1 - 2 files changed, 2 deletions(-) diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index ea451eea5..f5e978aeb 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -274,7 +274,6 @@ t_handle_msg_event(_) -> ok = meck:expect(emqx_cm, register_channel, fun(_, _, _) -> ok end), ok = meck:expect(emqx_cm, insert_channel_info, fun(_, _, _) -> ok end), ok = meck:expect(emqx_cm, set_chan_info, fun(_, _) -> ok end), - ok = meck:expect(emqx_cm, connection_closed, fun(_) -> ok end), ?assertEqual(ok, handle_msg({event, connected}, st())), ?assertMatch({ok, _St}, handle_msg({event, disconnected}, st())), ?assertMatch({ok, _St}, handle_msg({event, undefined}, st())). diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index 9860248fe..c494bddb2 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -483,7 +483,6 @@ t_handle_info_close(_) -> t_handle_info_event(_) -> ok = meck:expect(emqx_cm, register_channel, fun(_, _, _) -> ok end), ok = meck:expect(emqx_cm, insert_channel_info, fun(_, _, _) -> ok end), - ok = meck:expect(emqx_cm, connection_closed, fun(_) -> true end), {ok, _} = ?ws_conn:handle_info({event, connected}, st()), {ok, _} = ?ws_conn:handle_info({event, disconnected}, st()), {ok, _} = ?ws_conn:handle_info({event, updated}, st()). From fc37d235c7cfbfbfbfc45957f98a4bd227475213 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 25 Aug 2023 15:24:06 +0400 Subject: [PATCH 065/119] refactor(topicidx): simplify seek instructions Which also avoids comparing filter tail repeatedly when evaluating it. --- apps/emqx/src/emqx_trie_search.erl | 56 ++++++++++++++---------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/apps/emqx/src/emqx_trie_search.erl b/apps/emqx/src/emqx_trie_search.erl index 95174f292..445425cb8 100644 --- a/apps/emqx/src/emqx_trie_search.erl +++ b/apps/emqx/src/emqx_trie_search.erl @@ -137,7 +137,7 @@ get_topic({Filter, _ID}) when is_list(Filter) -> get_topic({Topic, _ID}) -> Topic. --compile({inline, [base/1, move_up/2, match_add/2, compare/3]}). +-compile({inline, [base/1, move_up/2, match_add/2, compare/4]}). %% Make the base-key which can be used to locate the desired search target. base(Prefix) -> @@ -210,31 +210,29 @@ search_new(Words0, NewBase, NextF, Acc) -> %% Search to the bigger end of ordered collection of topics and topic-filters. search_up(Words, {Filter, _} = Cursor, NextF, Acc) -> - case compare(Filter, Words, false) of + case compare(Filter, Words, 0, false) of match_full -> search_new(Words, Cursor, NextF, match_add(Cursor, Acc)); match_prefix -> search_new(Words, Cursor, NextF, Acc); lower -> {Cursor, Acc}; - [SeekWord | FilterTail] -> + {Pos, SeekWord} -> % NOTE - % This is a seek instruction. - % If we visualize the `Filter` as `FilterHead ++ [_] ++ FilterTail`, we need to - % seek to `FilterHead ++ [SeekWord]`. It carries the `FilterTail` because it's - % much cheaper to return it from `compare/3` than anything more usable. - NewBase = base(seek(SeekWord, Filter, FilterTail)), + % This is a seek instruction. It means we need to take `Pos` words + % from the current topic filter and attach `SeekWord` to the end of it. + NewBase = base(seek(Pos, SeekWord, Filter)), search_new(Words, NewBase, NextF, Acc) end. -seek(SeekWord, [_ | FilterTail], FilterTail) -> +seek(_Pos = 0, SeekWord, _FilterTail) -> [SeekWord]; -seek(SeekWord, [FilterWord | Rest], FilterTail) -> - [FilterWord | seek(SeekWord, Rest, FilterTail)]. +seek(Pos, SeekWord, [FilterWord | Rest]) -> + [FilterWord | seek(Pos - 1, SeekWord, Rest)]. -compare(NotFilter, _, _) when is_binary(NotFilter) -> +compare(NotFilter, _, _, _) when is_binary(NotFilter) -> lower; -compare([], [], _) -> +compare([], [], _, _) -> % NOTE % Topic: a/b/c/d % Filter: a/+/+/d @@ -244,7 +242,7 @@ compare([], [], _) -> % * a/+/+/d (same topic but a different ID) % * a/+/+/d/# (also a match) match_full; -compare([], _Words, _) -> +compare([], _Words, _, _) -> % NOTE % Topic: a/b/c/d % Filter: a/+/c @@ -257,7 +255,7 @@ compare([], _Words, _) -> % TODO % We might probably instead seek to a/+/c/# right away. match_prefix; -compare(['#'], _Words, _) -> +compare(['#'], _Words, _, _) -> % NOTE % Topic: a/b/c/d % Filter: a/+/+/d/# @@ -266,57 +264,57 @@ compare(['#'], _Words, _) -> % Closest possible next entries that we must not miss: % * a/+/+/d/# (same topic but a different ID) match_full; -compare(['+' | TF], [HW | TW], _PrevBacktrack) -> +compare(['+' | TF], [HW | TW], Pos, _PrevBacktrack) -> % NOTE % We need to keep backtrack point each time we encounter a plus. To safely skip over % parts of the search space, we may need last backtrack point when recursion terminates. % See next clauses for examples. - compare(TF, TW, [HW | TF]); -compare([HW | TF], [HW | TW], Backtrack) -> + compare(TF, TW, Pos + 1, {Pos, HW}); +compare([HW | TF], [HW | TW], Pos, Backtrack) -> % NOTE % Skip over the same word in both topic and filter, keeping the last backtrack point. - compare(TF, TW, Backtrack); -compare([HF | _], [HW | _], false) when HF > HW -> + compare(TF, TW, Pos + 1, Backtrack); +compare([HF | _], [HW | _], _, false) when HF > HW -> % NOTE % Topic: a/b/c/d % Filter: a/b/c/e/1 % The topic is lower than a topic filter. There's nowhere to backtrackto, we're out of % the search space. We should stop the search. lower; -compare([HF | _], [HW | _], Backtrack) when HF > HW -> +compare([HF | _], [HW | _], _, Backtrack) when HF > HW -> % NOTE % Topic: a/b/c/d % Filter: a/+/+/e/1 % The topic is lower than a topic filter. There was a plus, last time at the 3rd level, % we have a backtrack point to seek to: - % Seek: [c | e/1] + % Seek: {2, c} % We need to skip over part of search space, and seek to the next possible match: % Next: a/+/c Backtrack; -compare([_ | _], [], false) -> +compare([_ | _], [], _, false) -> % NOTE % Topic: a/b/c/d % Filter: a/b/c/d/1 - % The topic is lower than a topic filter. (since it's shorter). There's nowhere to + % The topic is lower than a topic filter (since it's shorter). There's nowhere to % backtrack to, we're out of the search space. We should stop the search. lower; -compare([_ | _], [], Backtrack) -> +compare([_ | _], [], _, Backtrack) -> % NOTE % Topic: a/b/c/d % Filter: a/+/c/d/1 % The topic is lower than a topic filter. There was a plus, last and only time at the % 3rd level, we have a backtrack point: - % Seek: [b | c/d/1] + % Seek: {1, b} % Next: a/b Backtrack; -compare([_HF | TF], [HW | _], _) -> +compare([_ | _], [HW | _], Pos, _) -> % NOTE % Topic: a/b/c/d % Filter: a/+/+/0/1/2 % Topic is higher than the filter, we need to skip over to the next possible filter. - % Seek: [d | 0/1/2] + % Seek: {3, d} % Next: a/+/+/d - [HW | TF]. + {Pos, HW}. match_add(K = {_Filter, ID}, Acc = #{}) -> % NOTE: ensuring uniqueness by record ID From 33ed53bb6aeab659c3d075838d24b7a04fcdfee7 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 25 Aug 2023 16:26:39 +0400 Subject: [PATCH 066/119] refactor(topicidx): simplify `compare/3` further It's possible to emit seek instruction just once, on the way back out of the `compare/3` stack. --- apps/emqx/src/emqx_trie_search.erl | 80 ++++++++++++++---------------- 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/apps/emqx/src/emqx_trie_search.erl b/apps/emqx/src/emqx_trie_search.erl index 445425cb8..4acee979f 100644 --- a/apps/emqx/src/emqx_trie_search.erl +++ b/apps/emqx/src/emqx_trie_search.erl @@ -137,7 +137,7 @@ get_topic({Filter, _ID}) when is_list(Filter) -> get_topic({Topic, _ID}) -> Topic. --compile({inline, [base/1, move_up/2, match_add/2, compare/4]}). +-compile({inline, [base/1, move_up/2, match_add/2, compare/3]}). %% Make the base-key which can be used to locate the desired search target. base(Prefix) -> @@ -210,7 +210,7 @@ search_new(Words0, NewBase, NextF, Acc) -> %% Search to the bigger end of ordered collection of topics and topic-filters. search_up(Words, {Filter, _} = Cursor, NextF, Acc) -> - case compare(Filter, Words, 0, false) of + case compare(Filter, Words, 0) of match_full -> search_new(Words, Cursor, NextF, match_add(Cursor, Acc)); match_prefix -> @@ -230,9 +230,9 @@ seek(_Pos = 0, SeekWord, _FilterTail) -> seek(Pos, SeekWord, [FilterWord | Rest]) -> [FilterWord | seek(Pos - 1, SeekWord, Rest)]. -compare(NotFilter, _, _, _) when is_binary(NotFilter) -> +compare(NotFilter, _, _) when is_binary(NotFilter) -> lower; -compare([], [], _, _) -> +compare([], [], _) -> % NOTE % Topic: a/b/c/d % Filter: a/+/+/d @@ -242,7 +242,7 @@ compare([], [], _, _) -> % * a/+/+/d (same topic but a different ID) % * a/+/+/d/# (also a match) match_full; -compare([], _Words, _, _) -> +compare([], _Words, _) -> % NOTE % Topic: a/b/c/d % Filter: a/+/c @@ -255,7 +255,7 @@ compare([], _Words, _, _) -> % TODO % We might probably instead seek to a/+/c/# right away. match_prefix; -compare(['#'], _Words, _, _) -> +compare(['#'], _Words, _) -> % NOTE % Topic: a/b/c/d % Filter: a/+/+/d/# @@ -264,50 +264,46 @@ compare(['#'], _Words, _, _) -> % Closest possible next entries that we must not miss: % * a/+/+/d/# (same topic but a different ID) match_full; -compare(['+' | TF], [HW | TW], Pos, _PrevBacktrack) -> - % NOTE - % We need to keep backtrack point each time we encounter a plus. To safely skip over - % parts of the search space, we may need last backtrack point when recursion terminates. - % See next clauses for examples. - compare(TF, TW, Pos + 1, {Pos, HW}); -compare([HW | TF], [HW | TW], Pos, Backtrack) -> +compare(['+' | TF], [HW | TW], Pos) -> + case compare(TF, TW, Pos + 1) of + lower -> + % NOTE + % Topic: a/b/c/d + % Filter: a/+/+/e/1 or a/b/+/d/1 + % The topic is lower than a topic filter. But we're at the `+` position, + % so we emit a backtrack point to seek to: + % Seek: {2, c} + % We skip over part of search space, and seek to the next possible match: + % Next: a/+/c + {Pos, HW}; + Other -> + % NOTE + % It's either already a backtrack point, emitted from the last `+` + % position or just a seek / match. In both cases we just pass it + % through. + Other + end; +compare([HW | TF], [HW | TW], Pos) -> % NOTE % Skip over the same word in both topic and filter, keeping the last backtrack point. - compare(TF, TW, Pos + 1, Backtrack); -compare([HF | _], [HW | _], _, false) when HF > HW -> + compare(TF, TW, Pos + 1); +compare([HF | _], [HW | _], _) when HF > HW -> % NOTE % Topic: a/b/c/d - % Filter: a/b/c/e/1 - % The topic is lower than a topic filter. There's nowhere to backtrackto, we're out of - % the search space. We should stop the search. + % Filter: a/b/c/e/1 or a/b/+/e + % The topic is lower than a topic filter. In the first case there's nowhere to + % backtrack to, we're out of the search space. In the second case there's a `+` + % on 3rd level, we'll seek up from there. lower; -compare([HF | _], [HW | _], _, Backtrack) when HF > HW -> +compare([_ | _], [], _) -> % NOTE % Topic: a/b/c/d - % Filter: a/+/+/e/1 - % The topic is lower than a topic filter. There was a plus, last time at the 3rd level, - % we have a backtrack point to seek to: - % Seek: {2, c} - % We need to skip over part of search space, and seek to the next possible match: - % Next: a/+/c - Backtrack; -compare([_ | _], [], _, false) -> - % NOTE - % Topic: a/b/c/d - % Filter: a/b/c/d/1 - % The topic is lower than a topic filter (since it's shorter). There's nowhere to - % backtrack to, we're out of the search space. We should stop the search. + % Filter: a/b/c/d/1 or a/+/c/d/1 + % The topic is lower than a topic filter (since it's shorter). In the first case + % there's nowhere to backtrack to, we're out of the search space. In the second case + % there's a `+` on 2nd level, we'll seek up from there. lower; -compare([_ | _], [], _, Backtrack) -> - % NOTE - % Topic: a/b/c/d - % Filter: a/+/c/d/1 - % The topic is lower than a topic filter. There was a plus, last and only time at the - % 3rd level, we have a backtrack point: - % Seek: {1, b} - % Next: a/b - Backtrack; -compare([_ | _], [HW | _], Pos, _) -> +compare([_ | _], [HW | _], Pos) -> % NOTE % Topic: a/b/c/d % Filter: a/+/+/0/1/2 From 558402a68e6cf244c7217578796daddd1f49639f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 25 Aug 2023 10:49:23 +0200 Subject: [PATCH 067/119] chore(topic_index): add topic validation --- apps/emqx/src/emqx_trie_search.erl | 22 +++++++++------- apps/emqx/test/emqx_trie_search_tests.erl | 32 +++++++++++++++++++++++ 2 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 apps/emqx/test/emqx_trie_search_tests.erl diff --git a/apps/emqx/src/emqx_trie_search.erl b/apps/emqx/src/emqx_trie_search.erl index 4acee979f..ddeef966b 100644 --- a/apps/emqx/src/emqx_trie_search.erl +++ b/apps/emqx/src/emqx_trie_search.erl @@ -113,7 +113,7 @@ %% @doc Make a search-key for the given topic. -spec make_key(emqx_types:topic(), ID) -> key(ID). make_key(Topic, ID) when is_binary(Topic) -> - Words = words(Topic), + Words = filter_words(Topic), case emqx_topic:wildcard(Words) of true -> %% it's a wildcard @@ -171,7 +171,7 @@ matches(Topic, NextF, Opts) -> %% @doc Entrypoint of the search for a given topic. search(Topic, NextF, Opts) -> - Words = words(Topic), + Words = topic_words(Topic), Base = base_init(Words), ORetFirst = proplists:get_bool(return_first, Opts), OUnique = proplists:get_bool(unique, Opts), @@ -320,18 +320,22 @@ match_add(K, Acc) when is_list(Acc) -> match_add(K, first) -> throw({first, K}). --spec words(emqx_types:topic()) -> [word()]. -words(Topic) when is_binary(Topic) -> +-spec filter_words(emqx_types:topic()) -> [word()]. +filter_words(Topic) when is_binary(Topic) -> % NOTE % This is almost identical to `emqx_topic:words/1`, but it doesn't convert empty % tokens to ''. This is needed to keep ordering of words consistent with what % `match_filter/3` expects. - [word(W) || W <- emqx_topic:tokens(Topic)]. + [word(W, filter) || W <- emqx_topic:tokens(Topic)]. --spec word(binary()) -> word(). -word(<<"+">>) -> '+'; -word(<<"#">>) -> '#'; -word(Bin) -> Bin. +topic_words(Topic) when is_binary(Topic) -> + [word(W, topic) || W <- emqx_topic:tokens(Topic)]. + +word(<<"+">>, topic) -> error(badarg); +word(<<"#">>, topic) -> error(badarg); +word(<<"+">>, filter) -> '+'; +word(<<"#">>, filter) -> '#'; +word(Bin, _) -> Bin. %% match non-wildcard topics match_topics(Topic, {Topic, _} = Key, NextF, Acc) -> diff --git a/apps/emqx/test/emqx_trie_search_tests.erl b/apps/emqx/test/emqx_trie_search_tests.erl new file mode 100644 index 000000000..75994131d --- /dev/null +++ b/apps/emqx/test/emqx_trie_search_tests.erl @@ -0,0 +1,32 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_trie_search_tests). + +-include_lib("eunit/include/eunit.hrl"). + +topic_validation_test() -> + NextF = fun(_) -> '$end_of_table' end, + Call = fun(Topic) -> + emqx_trie_search:match(Topic, NextF) + end, + ?assertError(badarg, Call(<<"+">>)), + ?assertError(badarg, Call(<<"#">>)), + ?assertError(badarg, Call(<<"a/+/b">>)), + ?assertError(badarg, Call(<<"a/b/#">>)), + ?assertEqual(false, Call(<<"a/b/b+">>)), + ?assertEqual(false, Call(<<"a/b/c#">>)), + ok. From ca59a87d47afacfba013e9e5c3b10e902422be78 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 25 Aug 2023 16:40:23 +0400 Subject: [PATCH 068/119] chore(topicidx): refine example of wildcard compare --- apps/emqx/src/emqx_trie_search.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_trie_search.erl b/apps/emqx/src/emqx_trie_search.erl index ddeef966b..6285eb4eb 100644 --- a/apps/emqx/src/emqx_trie_search.erl +++ b/apps/emqx/src/emqx_trie_search.erl @@ -258,7 +258,7 @@ compare([], _Words, _) -> compare(['#'], _Words, _) -> % NOTE % Topic: a/b/c/d - % Filter: a/+/+/d/# + % Filter: a/+/+/d/# or just a/# % We matched the topic to a topic filter with wildcard (possibly with pluses). % We include it in the result set, and now need to try next entry in the table. % Closest possible next entries that we must not miss: From d5cff533e3e3128bc36def697a217a0cc16f4a04 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 25 Aug 2023 16:41:21 +0400 Subject: [PATCH 069/119] chore(topicidx): drop TODO comment --- apps/emqx/src/emqx_trie_search.erl | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/emqx/src/emqx_trie_search.erl b/apps/emqx/src/emqx_trie_search.erl index 6285eb4eb..b774e1459 100644 --- a/apps/emqx/src/emqx_trie_search.erl +++ b/apps/emqx/src/emqx_trie_search.erl @@ -251,9 +251,6 @@ compare([], _Words, _) -> % Closest possible next entries that we must not miss: % * a/+/c/# (which is a match) % * a/+/c/+ (also a match) - % - % TODO - % We might probably instead seek to a/+/c/# right away. match_prefix; compare(['#'], _Words, _) -> % NOTE From 42877e282df251e3d509457658c3ca141f5c63a7 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Fri, 25 Aug 2023 13:53:52 -0300 Subject: [PATCH 070/119] fix: flatten error message on resource validator Fixes https://emqx.atlassian.net/browse/EMQX-10864 --- .../src/emqx_resource_validator.erl | 13 ++++- .../test/emqx_resource_SUITE.erl | 50 ++++++++++++++++++- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_validator.erl b/apps/emqx_resource/src/emqx_resource_validator.erl index bc733ef80..ea9547265 100644 --- a/apps/emqx_resource/src/emqx_resource_validator.erl +++ b/apps/emqx_resource/src/emqx_resource_validator.erl @@ -28,10 +28,18 @@ max(Type, Max) -> min(Type, Min) -> limit(Type, '>=', Min). -not_empty(ErrMsg) -> +not_empty(ErrMsg0) -> + ErrMsg = + try + lists:flatten(ErrMsg0) + catch + _:_ -> + ErrMsg0 + end, fun (undefined) -> {error, ErrMsg}; (<<>>) -> {error, ErrMsg}; + ("") -> {error, ErrMsg}; (_) -> ok end. @@ -50,7 +58,8 @@ len(string) -> fun string:length/1; len(_Type) -> fun(Val) -> Val end. err_limit({Type, {Op, Expected}, {got, Got}}) -> - io_lib:format("Expect the ~ts value ~ts ~p but got: ~p", [Type, Op, Expected, Got]). + Msg = io_lib:format("Expect the ~ts value ~ts ~p but got: ~p", [Type, Op, Expected, Got]), + lists:flatten(Msg). return(true, _) -> ok; return(false, Error) -> {error, Error}. diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 934a97829..77e680933 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1121,10 +1121,58 @@ t_create_dry_run_local_failed(_) -> ). t_test_func(_) -> + IsErrorMsgPlainString = fun({error, Msg}) -> io_lib:printable_list(Msg) end, ?assertEqual(ok, erlang:apply(emqx_resource_validator:not_empty("not_empty"), [<<"someval">>])), ?assertEqual(ok, erlang:apply(emqx_resource_validator:min(int, 3), [4])), ?assertEqual(ok, erlang:apply(emqx_resource_validator:max(array, 10), [[a, b, c, d]])), - ?assertEqual(ok, erlang:apply(emqx_resource_validator:max(string, 10), ["less10"])). + ?assertEqual(ok, erlang:apply(emqx_resource_validator:max(string, 10), ["less10"])), + ?assertEqual( + true, IsErrorMsgPlainString(erlang:apply(emqx_resource_validator:min(int, 66), [42])) + ), + ?assertEqual( + true, IsErrorMsgPlainString(erlang:apply(emqx_resource_validator:max(int, 42), [66])) + ), + ?assertEqual( + true, IsErrorMsgPlainString(erlang:apply(emqx_resource_validator:min(array, 3), [[1, 2]])) + ), + ?assertEqual( + true, + IsErrorMsgPlainString(erlang:apply(emqx_resource_validator:max(array, 3), [[1, 2, 3, 4]])) + ), + ?assertEqual( + true, IsErrorMsgPlainString(erlang:apply(emqx_resource_validator:min(string, 3), ["1"])) + ), + ?assertEqual( + true, IsErrorMsgPlainString(erlang:apply(emqx_resource_validator:max(string, 3), ["1234"])) + ), + NestedMsg = io_lib:format("The answer: ~p", [42]), + ExpectedMsg = "The answer: 42", + BinMsg = <<"The answer: 42">>, + MapMsg = #{question => "The question", answer => 42}, + ?assertEqual( + {error, ExpectedMsg}, + erlang:apply(emqx_resource_validator:not_empty(NestedMsg), [""]) + ), + ?assertEqual( + {error, ExpectedMsg}, + erlang:apply(emqx_resource_validator:not_empty(NestedMsg), [<<>>]) + ), + ?assertEqual( + {error, ExpectedMsg}, + erlang:apply(emqx_resource_validator:not_empty(NestedMsg), [undefined]) + ), + ?assertEqual( + {error, ExpectedMsg}, + erlang:apply(emqx_resource_validator:not_empty(NestedMsg), [undefined]) + ), + ?assertEqual( + {error, BinMsg}, + erlang:apply(emqx_resource_validator:not_empty(BinMsg), [undefined]) + ), + ?assertEqual( + {error, MapMsg}, + erlang:apply(emqx_resource_validator:not_empty(MapMsg), [""]) + ). t_reset_metrics(_) -> {ok, _} = emqx_resource:create( From d8be248a3dcb51c6ba100ccfeeace81c23424299 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 25 Aug 2023 17:20:02 +0800 Subject: [PATCH 071/119] fix: improve the suggest msg for update conf failed --- apps/emqx_conf/src/emqx_conf_cli.erl | 123 +++++++++++------- .../src/emqx_mgmt_api_configs.erl | 2 +- 2 files changed, 76 insertions(+), 49 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 8a4bb131f..e07963c09 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -50,17 +50,17 @@ conf(["show"]) -> conf(["show", Key]) -> print_hocon(get_config(list_to_binary(Key))); conf(["load", "--replace", Path]) -> - load_config(Path, replace); + load_config(Path, #{mode => replace}); conf(["load", "--merge", Path]) -> - load_config(Path, merge); + load_config(Path, #{mode => merge}); conf(["load", Path]) -> - load_config(Path, merge); + load_config(Path, #{mode => merge}); conf(["cluster_sync" | Args]) -> admins(Args); conf(["reload", "--merge"]) -> - reload_etc_conf_on_local_node(merge); + reload_etc_conf_on_local_node(#{mode => merge}); conf(["reload", "--replace"]) -> - reload_etc_conf_on_local_node(replace); + reload_etc_conf_on_local_node(#{mode => replace}); conf(["reload"]) -> conf(["reload", "--merge"]); conf(_) -> @@ -191,32 +191,32 @@ get_config(Key) -> end. -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}). -load_config(Path, ReplaceOrMerge) when is_list(Path) -> +load_config(Path, Opts) when is_list(Path) -> case hocon:files([Path]) of {ok, RawConf} when RawConf =:= #{} -> emqx_ctl:warning("load ~ts is empty~n", [Path]), {error, empty_hocon_file}; {ok, RawConf} -> - load_config_from_raw(RawConf, ReplaceOrMerge); + load_config_from_raw(RawConf, Opts); {error, Reason} -> emqx_ctl:warning("load ~ts failed~n~p~n", [Path, Reason]), {error, bad_hocon_file} end; -load_config(Bin, ReplaceOrMerge) when is_binary(Bin) -> +load_config(Bin, Opts) when is_binary(Bin) -> case hocon:binary(Bin) of {ok, RawConf} -> - load_config_from_raw(RawConf, ReplaceOrMerge); + load_config_from_raw(RawConf, Opts); {error, Reason} -> {error, Reason} end. -load_config_from_raw(RawConf, ReplaceOrMerge) -> +load_config_from_raw(RawConf, Opts) -> case check_config(RawConf) of ok -> Error = lists:filtermap( fun({K, V}) -> - case update_config_cluster(K, V, ReplaceOrMerge) of + case update_config_cluster(K, V, Opts) of ok -> false; {error, Msg} -> {true, Msg} end @@ -228,53 +228,71 @@ load_config_from_raw(RawConf, ReplaceOrMerge) -> ErrorBin -> {error, ErrorBin} end; {error, ?UPDATE_READONLY_KEYS_PROHIBITED = Reason} -> - emqx_ctl:warning("load config failed~n~ts~n", [Reason]), - emqx_ctl:warning( - "Maybe try `emqx_ctl conf reload` to reload etc/emqx.conf on local node~n" + warning(Opts, "load config failed~n~ts~n", [Reason]), + warning( + Opts, + "Maybe try `emqx_ctl conf reload` to reload etc/emqx.conf on local node~n", + [] ), {error, Reason}; {error, Errors} -> - emqx_ctl:warning("load schema check failed~n"), + warning(Opts, "load schema check failed~n", []), lists:foreach( fun({Key, Error}) -> - emqx_ctl:warning("~ts: ~p~n", [Key, Error]) + warning(Opts, "~ts: ~p~n", [Key, Error]) end, Errors ), {error, Errors} end. -update_config_cluster(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge = Mode) -> - check_res(Key, emqx_authz:merge(Conf), Conf, Mode); -update_config_cluster(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge = Mode) -> - check_res(Key, emqx_authn:merge_config(Conf), Conf, Mode); -update_config_cluster(Key, NewConf, merge = Mode) -> +update_config_cluster( + ?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY = Key, + Conf, + #{mode := merge} = Opts +) -> + check_res(Key, emqx_authz:merge(Conf), Conf, Opts); +update_config_cluster( + ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY = Key, + Conf, + #{mode := merge} = Opts +) -> + check_res(Key, emqx_authn:merge_config(Conf), Conf, Opts); +update_config_cluster(Key, NewConf, #{mode := merge} = Opts) -> Merged = merge_conf(Key, NewConf), - check_res(Key, emqx_conf:update([Key], Merged, ?OPTIONS), NewConf, Mode); -update_config_cluster(Key, Value, replace = Mode) -> - check_res(Key, emqx_conf:update([Key], Value, ?OPTIONS), Value, Mode). + check_res(Key, emqx_conf:update([Key], Merged, ?OPTIONS), NewConf, Opts); +update_config_cluster(Key, Value, #{mode := replace} = Opts) -> + check_res(Key, emqx_conf:update([Key], Value, ?OPTIONS), Value, Opts). -define(LOCAL_OPTIONS, #{rawconf_with_defaults => true, persistent => false}). -update_config_local(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge = Mode) -> - check_res(node(), Key, emqx_authz:merge_local(Conf, ?LOCAL_OPTIONS), Conf, Mode); -update_config_local(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge = Mode) -> - check_res(node(), Key, emqx_authn:merge_config_local(Conf, ?LOCAL_OPTIONS), Conf, Mode); -update_config_local(Key, NewConf, merge = Mode) -> +update_config_local( + ?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY = Key, + Conf, + #{mode := merge} = Opts +) -> + check_res(node(), Key, emqx_authz:merge_local(Conf, ?LOCAL_OPTIONS), Conf, Opts); +update_config_local( + ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY = Key, + Conf, + #{mode := merge} = Opts +) -> + check_res(node(), Key, emqx_authn:merge_config_local(Conf, ?LOCAL_OPTIONS), Conf, Opts); +update_config_local(Key, NewConf, #{mode := merge} = Opts) -> Merged = merge_conf(Key, NewConf), - check_res(node(), Key, emqx:update_config([Key], Merged, ?LOCAL_OPTIONS), NewConf, Mode); -update_config_local(Key, Value, replace = Mode) -> - check_res(node(), Key, emqx:update_config([Key], Value, ?LOCAL_OPTIONS), Value, Mode). + check_res(node(), Key, emqx:update_config([Key], Merged, ?LOCAL_OPTIONS), NewConf, Opts); +update_config_local(Key, Value, #{mode := replace} = Opts) -> + check_res(node(), Key, emqx:update_config([Key], Value, ?LOCAL_OPTIONS), Value, Opts). -check_res(Key, Res, Conf, Mode) -> check_res(cluster, Key, Res, Conf, Mode). -check_res(Node, Key, {ok, _}, _Conf, _Mode) -> - emqx_ctl:print("load ~ts on ~p ok~n", [Key, Node]), +check_res(Key, Res, Conf, Opts) -> check_res(cluster, Key, Res, Conf, Opts). +check_res(Node, Key, {ok, _}, _Conf, Opts) -> + print(Opts, "load ~ts on ~p ok~n", [Key, Node]), ok; -check_res(_Node, Key, {error, Reason}, Conf, Mode) -> +check_res(_Node, Key, {error, Reason}, Conf, Opts = #{mode := Mode}) -> Warning = "Can't ~ts the new configurations!~n" "Root key: ~ts~n" "Reason: ~p~n", - emqx_ctl:warning(Warning, [Mode, Key, Reason]), + warning(Opts, Warning, [Mode, Key, Reason]), ActiveMsg0 = "The effective configurations:~n" "```~n" @@ -285,12 +303,13 @@ check_res(_Node, Key, {error, Reason}, Conf, Mode) -> "```~n" "~ts```~n", FailedMsg = io_lib:format(FailedMsg0, [Mode, hocon_pp:do(#{Key => Conf}, #{})]), - SuggestMsg = suggest_msg(Mode), + SuggestMsg = suggest_msg(Reason, Mode), Msg = iolist_to_binary([ActiveMsg, FailedMsg, SuggestMsg]), - emqx_ctl:print("~ts", [Msg]), + print(Opts, "~ts", [Msg]), {error, iolist_to_binary([Warning, Msg])}. -suggest_msg(Mode) when Mode == merge orelse Mode == replace -> +%% The mix data failed validation, suggest the user to retry with another mode. +suggest_msg(#{kind := validation_error, reason := unknown_fields}, Mode) -> RetryMode = case Mode of merge -> "replace"; @@ -300,7 +319,9 @@ suggest_msg(Mode) when Mode == merge orelse Mode == replace -> "Tips: There may be some conflicts in the new configuration under `~ts` mode,~n" "Please retry with the `~ts` mode.~n", [Mode, RetryMode] - ). + ); +suggest_msg(_, _) -> + <<"">>. check_config(Conf) -> case check_keys_is_not_readonly(Conf) of @@ -327,19 +348,19 @@ check_config_schema(Conf) -> sorted_fold(Fold, Conf). %% @doc Reload etc/emqx.conf to runtime config except for the readonly config --spec reload_etc_conf_on_local_node(replace | merge) -> ok | {error, term()}. -reload_etc_conf_on_local_node(ReplaceOrMerge) -> +-spec reload_etc_conf_on_local_node(#{mode => replace | merge}) -> ok | {error, term()}. +reload_etc_conf_on_local_node(Opts) -> case load_etc_config_file() of {ok, RawConf} -> case filter_readonly_config(RawConf) of {ok, Reloaded} -> - reload_config(Reloaded, ReplaceOrMerge); + reload_config(Reloaded, Opts); {error, Error} -> - emqx_ctl:warning("check config failed~n~p~n", [Error]), + warning(Opts, "check config failed~n~p~n", [Error]), {error, Error} end; {error, Error} -> - emqx_ctl:warning("bad_hocon_file~n ~p~n", [Error]), + warning(Opts, "bad_hocon_file~n ~p~n", [Error]), {error, bad_hocon_file} end. @@ -385,9 +406,9 @@ filter_readonly_config(Raw) -> {error, Error} end. -reload_config(AllConf, ReplaceOrMerge) -> +reload_config(AllConf, Opts) -> Fold = fun({Key, Conf}, Acc) -> - case update_config_local(Key, Conf, ReplaceOrMerge) of + case update_config_local(Key, Conf, Opts) of ok -> Acc; Error -> @@ -441,3 +462,9 @@ check_config(SchemaMod, Key, Value) -> throw:Error -> {error, Error} end. + +warning(#{log := none}, _, _) -> ok; +warning(_, Format, Args) -> emqx_ctl:warning(Format, Args). + +print(#{log := none}, _, _) -> ok; +print(_, Format, Args) -> emqx_ctl:print(Format, Args). diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 5edf8c564..5fe08a0e4 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -344,7 +344,7 @@ configs(get, #{query_string := QueryStr, headers := Headers}, _Req) -> {error, _} = Error -> {400, #{code => 'INVALID_ACCEPT', message => ?ERR_MSG(Error)}} end; configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) -> - case emqx_conf_cli:load_config(Conf, Mode) of + case emqx_conf_cli:load_config(Conf, #{mode => Mode, log => none}) of ok -> {200}; {error, Msg} -> {400, #{<<"content-type">> => <<"text/plain">>}, Msg} end. From 053b7fb94ab35f0c631f0f12a018c96666ca2aaa Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 28 Aug 2023 11:14:36 +0800 Subject: [PATCH 072/119] test: add more test for conf_cli --- apps/emqx_conf/src/emqx_conf_cli.erl | 8 +- apps/emqx_conf/test/emqx_conf_cli_SUITE.erl | 86 ++++++++++++++++++++- 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index e07963c09..cdf188001 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -292,7 +292,6 @@ check_res(_Node, Key, {error, Reason}, Conf, Opts = #{mode := Mode}) -> "Can't ~ts the new configurations!~n" "Root key: ~ts~n" "Reason: ~p~n", - warning(Opts, Warning, [Mode, Key, Reason]), ActiveMsg0 = "The effective configurations:~n" "```~n" @@ -305,8 +304,9 @@ check_res(_Node, Key, {error, Reason}, Conf, Opts = #{mode := Mode}) -> FailedMsg = io_lib:format(FailedMsg0, [Mode, hocon_pp:do(#{Key => Conf}, #{})]), SuggestMsg = suggest_msg(Reason, Mode), Msg = iolist_to_binary([ActiveMsg, FailedMsg, SuggestMsg]), - print(Opts, "~ts", [Msg]), - {error, iolist_to_binary([Warning, Msg])}. + print(Opts, "~ts~n", [Msg]), + warning(Opts, Warning, [Mode, Key, Reason]), + {error, iolist_to_binary([Msg, "\n", io_lib:format(Warning, [Mode, Key, Reason])])}. %% The mix data failed validation, suggest the user to retry with another mode. suggest_msg(#{kind := validation_error, reason := unknown_fields}, Mode) -> @@ -317,7 +317,7 @@ suggest_msg(#{kind := validation_error, reason := unknown_fields}, Mode) -> end, io_lib:format( "Tips: There may be some conflicts in the new configuration under `~ts` mode,~n" - "Please retry with the `~ts` mode.~n", + "Please retry with the `~ts` mode.", [Mode, RetryMode] ); suggest_msg(_, _) -> diff --git a/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl index c7701b431..e5356c2ea 100644 --- a/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl @@ -27,11 +27,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authz]), + emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authz, emqx_authn]), Config. end_per_suite(_Config) -> - emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_authz]). + emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_authz, emqx_authn]). t_load_config(Config) -> Authz = authorization, @@ -64,6 +64,88 @@ t_load_config(Config) -> ?assertEqual({error, empty_hocon_file}, emqx_conf_cli:conf(["load", "non-exist-file"])), ok. +t_conflict_mix_conf(Config) -> + case emqx_release:edition() of + ce -> + %% Don't fail if the test is run with emqx profile + ok; + ee -> + AuthNInit = emqx_conf:get_raw([authentication]), + Redis = #{ + <<"backend">> => <<"redis">>, + <<"cmd">> => <<"HMGET mqtt_user:${username} password_hash salt">>, + <<"enable">> => false, + <<"mechanism">> => <<"password_based">>, + %% password_hash_algorithm {name = sha256, salt_position = suffix} + <<"redis_type">> => <<"single">>, + <<"server">> => <<"127.0.0.1:6379">> + }, + AuthN = #{<<"authentication">> => [Redis]}, + ConfBin = hocon_pp:do(AuthN, #{}), + ConfFile = prepare_conf_file(?FUNCTION_NAME, ConfBin, Config), + %% init with redis sources + ok = emqx_conf_cli:conf(["load", "--replace", ConfFile]), + ?assertMatch([Redis], emqx_conf:get_raw([authentication])), + %% change redis type from single to cluster + %% the server field will become servers field + RedisCluster = maps:remove(<<"server">>, Redis#{ + <<"redis_type">> => cluster, + <<"servers">> => [<<"127.0.0.1:6379">>] + }), + AuthN1 = AuthN#{<<"authentication">> => [RedisCluster]}, + ConfBin1 = hocon_pp:do(AuthN1, #{}), + ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config), + {error, Reason} = emqx_conf_cli:conf(["load", "--merge", ConfFile1]), + ?assertNotEqual( + nomatch, + binary:match( + Reason, + [<<"Tips: There may be some conflicts in the new configuration under">>] + ), + Reason + ), + %% use replace to change redis type from single to cluster + ?assertMatch(ok, emqx_conf_cli:conf(["load", "--replace", ConfFile1])), + %% clean up + ConfBinInit = hocon_pp:do(#{<<"authentication">> => AuthNInit}, #{}), + ConfFileInit = prepare_conf_file(?FUNCTION_NAME, ConfBinInit, Config), + ok = emqx_conf_cli:conf(["load", "--replace", ConfFileInit]), + ok + end. + +t_config_handler_hook_failed(Config) -> + Listeners = + #{ + <<"listeners">> => #{ + <<"ssl">> => #{ + <<"default">> => #{ + <<"ssl_options">> => #{ + <<"keyfile">> => <<"">> + } + } + } + } + }, + ConfBin = hocon_pp:do(Listeners, #{}), + ConfFile = prepare_conf_file(?FUNCTION_NAME, ConfBin, Config), + {error, Reason} = emqx_conf_cli:conf(["load", "--merge", ConfFile]), + %% the hook failed with empty keyfile + ?assertEqual( + nomatch, + binary:match(Reason, [ + <<"Tips: There may be some conflicts in the new configuration under">> + ]), + Reason + ), + ?assertNotEqual( + nomatch, + binary:match(Reason, [ + <<"{bad_ssl_config,#{reason => pem_file_path_or_string_is_required">> + ]), + Reason + ), + ok. + t_load_readonly(Config) -> Base0 = base_conf(), Base1 = Base0#{<<"mqtt">> => emqx_conf:get_raw([mqtt])}, From fe37842b11bec4f692df1b5d0152885c607fdc32 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 28 Aug 2023 11:27:32 +0800 Subject: [PATCH 073/119] chore: update 11523 changelog --- changes/ce/fix-11523.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-11523.en.md diff --git a/changes/ce/fix-11523.en.md b/changes/ce/fix-11523.en.md new file mode 100644 index 000000000..d8b6d8568 --- /dev/null +++ b/changes/ce/fix-11523.en.md @@ -0,0 +1 @@ +Fixes misunderstood prompt when invalid certificates/keys were specified for the `/configs` API. From f3467f44b2eaf6ec7f0d403fac2abc427e5a0174 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 28 Aug 2023 17:21:58 +0800 Subject: [PATCH 074/119] chore: bump hstreamdb_erl driver to `0.4.5+v0.16.1` --- apps/emqx_bridge_hstreamdb/rebar.config | 2 +- changes/ee/feat-11478.en.md | 2 ++ mix.exs | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_hstreamdb/rebar.config b/apps/emqx_bridge_hstreamdb/rebar.config index 44970b75d..fb99cd627 100644 --- a/apps/emqx_bridge_hstreamdb/rebar.config +++ b/apps/emqx_bridge_hstreamdb/rebar.config @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {erl_opts, [debug_info]}. {deps, [ - {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.4.3+v0.16.1"}}}, + {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.4.5+v0.16.1"}}}, {emqx, {path, "../../apps/emqx"}}, {emqx_utils, {path, "../../apps/emqx_utils"}} ]}. diff --git a/changes/ee/feat-11478.en.md b/changes/ee/feat-11478.en.md index 42761c77e..b2d1a49a2 100644 --- a/changes/ee/feat-11478.en.md +++ b/changes/ee/feat-11478.en.md @@ -1 +1,3 @@ Add HStreamDB bridge support (both TCP and TLS connection allowed), adapted to the HStreamDB `v0.16.1`. + +Updated driver to `0.4.5+v0.16.1` in [PR#11530](https://github.com/emqx/emqx/pull/11530). diff --git a/mix.exs b/mix.exs index 496f0b0c5..3aee37a3d 100644 --- a/mix.exs +++ b/mix.exs @@ -227,7 +227,7 @@ defmodule EMQXUmbrella.MixProject do defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do [ - {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.4.3+v0.16.1"}, + {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.4.5+v0.16.1"}, {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.11", override: true}, {:wolff, github: "kafka4beam/wolff", tag: "1.7.6"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.3", override: true}, From 9eccfa09093dc798229fcaea757dd3308fbcd807 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 7 Aug 2023 22:28:37 +0400 Subject: [PATCH 075/119] refactor(router): isolate cleanup logic in router module --- apps/emqx/src/emqx_router.erl | 16 ++++++++++++++++ apps/emqx/src/emqx_router_helper.erl | 9 +-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 95b6136a7..fcb9b4e45 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -46,6 +46,8 @@ do_delete_route/2 ]). +-export([cleanup_routes/1]). + -export([ match_routes/1, lookup_routes/1, @@ -70,6 +72,8 @@ -type dest() :: node() | {group(), node()}. +-dialyzer({nowarn_function, [cleanup_routes/1]}). + %%-------------------------------------------------------------------- %% Mnesia bootstrap %%-------------------------------------------------------------------- @@ -196,6 +200,18 @@ print_routes(Topic) -> match_routes(Topic) ). +-spec cleanup_routes(node()) -> ok. +cleanup_routes(Node) -> + Patterns = [ + #route{_ = '_', dest = Node}, + #route{_ = '_', dest = {'_', Node}} + ], + [ + mnesia:delete_object(?ROUTE_TAB, Route, write) + || Pat <- Patterns, + Route <- mnesia:match_object(?ROUTE_TAB, Pat, write) + ]. + call(Router, Msg) -> gen_server:call(Router, Msg, infinity). diff --git a/apps/emqx/src/emqx_router_helper.erl b/apps/emqx/src/emqx_router_helper.erl index 8d96bf81d..61573fcff 100644 --- a/apps/emqx/src/emqx_router_helper.erl +++ b/apps/emqx/src/emqx_router_helper.erl @@ -197,11 +197,4 @@ stats_fun() -> end. cleanup_routes(Node) -> - Patterns = [ - #route{_ = '_', dest = Node}, - #route{_ = '_', dest = {'_', Node}} - ], - [ - mnesia:delete_object(?ROUTE_TAB, Route, write) - || Pat <- Patterns, Route <- mnesia:match_object(?ROUTE_TAB, Pat, write) - ]. + emqx_router:cleanup_routes(Node). From dcb63440bcdcd42ad5e8a74d1921607ef4f79666 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 8 Aug 2023 15:34:04 +0400 Subject: [PATCH 076/119] refactor(mgmt): avoid dealing with router tab directly Instead, contain all this behind `emqx_router` module interface. --- apps/emqx/src/emqx_router.erl | 13 ++++++++++++ apps/emqx_management/src/emqx_mgmt_cli.erl | 24 +++++----------------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index fcb9b4e45..a403efa84 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -56,6 +56,11 @@ -export([print_routes/1]). +-export([ + foldl_routes/2, + foldr_routes/2 +]). + -export([topics/0]). %% gen_server callbacks @@ -212,6 +217,14 @@ cleanup_routes(Node) -> Route <- mnesia:match_object(?ROUTE_TAB, Pat, write) ]. +-spec foldl_routes(fun((emqx_types:route(), Acc) -> Acc), Acc) -> Acc. +foldl_routes(FoldFun, AccIn) -> + ets:foldl(FoldFun, AccIn, ?ROUTE_TAB). + +-spec foldr_routes(fun((emqx_types:route(), Acc) -> Acc), Acc) -> Acc. +foldr_routes(FoldFun, AccIn) -> + ets:foldr(FoldFun, AccIn, ?ROUTE_TAB). + call(Router, Msg) -> gen_server:call(Router, Msg, infinity). diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index aeed5b922..121e5b1a8 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -22,9 +22,6 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). --include("emqx_mgmt.hrl"). - --define(PRINT_CMD(Cmd, Descr), io:format("~-48s# ~ts~n", [Cmd, Descr])). -define(DATA_BACKUP_OPTS, #{print_fun => fun emqx_ctl:print/2}). -export([load/0]). @@ -49,20 +46,6 @@ data/1 ]). --define(PROC_INFOKEYS, [ - status, - memory, - message_queue_len, - total_heap_size, - heap_size, - stack_size, - reductions -]). - --define(MAX_LIMIT, 10000). - --define(APP, emqx). - -spec load() -> ok. load() -> Cmds = [Fun || {Fun, _} <- ?MODULE:module_info(exports), is_cmd(Fun)], @@ -197,9 +180,12 @@ if_client(ClientId, Fun) -> %% @doc Topics Command topics(["list"]) -> - dump(?ROUTE_TAB, emqx_topic); + emqx_router:foldr_routes( + fun(Route, Acc) -> [print({emqx_topic, Route}) | Acc] end, + [] + ); topics(["show", Topic]) -> - Routes = ets:lookup(?ROUTE_TAB, bin(Topic)), + Routes = emqx_router:lookup_routes(Topic), [print({emqx_topic, Route}) || Route <- Routes]; topics(_) -> emqx_ctl:usage([ From 84e40fb6fef90a1dc1466091e6a417b48c41cedf Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 15 Aug 2023 11:18:22 +0400 Subject: [PATCH 077/119] test(trie): add more involved route add + delete case That shows how current local trie implementation breaks because of lack of refcounting. --- apps/emqx/test/emqx_router_SUITE.erl | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/apps/emqx/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl index 067f11634..453f86257 100644 --- a/apps/emqx/test/emqx_router_SUITE.erl +++ b/apps/emqx/test/emqx_router_SUITE.erl @@ -79,6 +79,55 @@ t_add_delete(_) -> ?R:delete_route(<<"a/+/b">>, node()), ?assertEqual([], ?R:topics()). +t_add_delete_incremental(_) -> + ?R:add_route(<<"a/b/c">>), + ?R:add_route(<<"a/+/c">>, node()), + ?R:add_route(<<"a/+/+">>, node()), + ?R:add_route(<<"a/b/#">>, node()), + ?R:add_route(<<"#">>, node()), + ?assertEqual( + [ + #route{topic = <<"#">>, dest = node()}, + #route{topic = <<"a/+/+">>, dest = node()}, + #route{topic = <<"a/+/c">>, dest = node()}, + #route{topic = <<"a/b/#">>, dest = node()}, + #route{topic = <<"a/b/c">>, dest = node()} + ], + lists:sort(?R:match_routes(<<"a/b/c">>)) + ), + ?R:delete_route(<<"a/+/c">>, node()), + ?assertEqual( + [ + #route{topic = <<"#">>, dest = node()}, + #route{topic = <<"a/+/+">>, dest = node()}, + #route{topic = <<"a/b/#">>, dest = node()}, + #route{topic = <<"a/b/c">>, dest = node()} + ], + lists:sort(?R:match_routes(<<"a/b/c">>)) + ), + ?R:delete_route(<<"a/+/+">>, node()), + ?assertEqual( + [ + #route{topic = <<"#">>, dest = node()}, + #route{topic = <<"a/b/#">>, dest = node()}, + #route{topic = <<"a/b/c">>, dest = node()} + ], + lists:sort(?R:match_routes(<<"a/b/c">>)) + ), + ?R:delete_route(<<"a/b/#">>, node()), + ?assertEqual( + [ + #route{topic = <<"#">>, dest = node()}, + #route{topic = <<"a/b/c">>, dest = node()} + ], + lists:sort(?R:match_routes(<<"a/b/c">>)) + ), + ?R:delete_route(<<"a/b/c">>, node()), + ?assertEqual( + [#route{topic = <<"#">>, dest = node()}], + lists:sort(?R:match_routes(<<"a/b/c">>)) + ). + t_do_add_delete(_) -> ?R:do_add_route(<<"a/b/c">>), ?R:do_add_route(<<"a/b/c">>, node()), From 166375a0003a31d98a3bda9e6a4732bb4401dc5a Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 16 Aug 2023 15:01:31 +0400 Subject: [PATCH 078/119] fix(topicidx): make `get_record/2` simpler to use in concurrent env The mechanic of `emqx_topic_index` cannot really guarantee atomicity of reading records associated with index matches, so instead it's probably better to make the user aware of that lack of this guarantee. --- apps/emqx/src/emqx_topic_index.erl | 12 +++++++++--- apps/emqx_rule_engine/src/emqx_rule_engine.erl | 5 +++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_topic_index.erl b/apps/emqx/src/emqx_topic_index.erl index acbf36627..0b153ac01 100644 --- a/apps/emqx/src/emqx_topic_index.erl +++ b/apps/emqx/src/emqx_topic_index.erl @@ -73,10 +73,16 @@ get_topic(Key) -> emqx_trie_search:get_topic(Key). %% @doc Fetch the record associated with the match. -%% NOTE: Only really useful for ETS tables where the record ID is the first element. --spec get_record(match(_ID), ets:table()) -> _Record. +%% May return empty list if the index entry was deleted in the meantime. +%% NOTE: Only really useful for ETS tables where the record data is the last element. +-spec get_record(match(_ID), ets:table()) -> [_Record]. get_record(K, Tab) -> - ets:lookup_element(Tab, K, 2). + case ets:lookup(Tab, K) of + [Entry] -> + [erlang:element(tuple_size(Entry), Entry)]; + [] -> + [] + end. key(TopicOrFilter, ID) -> emqx_trie_search:make_key(TopicOrFilter, ID). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 41d1ed433..0060ed819 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -225,8 +225,9 @@ get_rules_ordered_by_ts() -> -spec get_rules_for_topic(Topic :: binary()) -> [rule()]. get_rules_for_topic(Topic) -> [ - emqx_topic_index:get_record(M, ?RULE_TOPIC_INDEX) - || M <- emqx_topic_index:matches(Topic, ?RULE_TOPIC_INDEX, [unique]) + Rule + || M <- emqx_topic_index:matches(Topic, ?RULE_TOPIC_INDEX, [unique]), + Rule <- emqx_topic_index:get_record(M, ?RULE_TOPIC_INDEX) ]. -spec get_rules_with_same_event(Topic :: binary()) -> [rule()]. From 33e5e1ba5765d8f3d632ca24471633db71d8b64a Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 25 Aug 2023 14:49:01 +0400 Subject: [PATCH 079/119] feat(router): add unified routing table --- apps/emqx/include/emqx_router.hrl | 3 +- apps/emqx/src/emqx_router.erl | 221 +++++++++++++++++--- apps/emqx/src/emqx_router_helper.erl | 8 +- apps/emqx/src/emqx_topic_index.erl | 13 +- apps/emqx/test/emqx_router_SUITE.erl | 45 ++-- apps/emqx/test/emqx_router_helper_SUITE.erl | 70 ++++--- 6 files changed, 280 insertions(+), 80 deletions(-) diff --git a/apps/emqx/include/emqx_router.hrl b/apps/emqx/include/emqx_router.hrl index 035ff5455..99ca3e185 100644 --- a/apps/emqx/include/emqx_router.hrl +++ b/apps/emqx/include/emqx_router.hrl @@ -17,8 +17,9 @@ -ifndef(EMQX_ROUTER_HRL). -define(EMQX_ROUTER_HRL, true). -%% ETS table for message routing +%% ETS tables for message routing -define(ROUTE_TAB, emqx_route). +-define(ROUTE_TAB_UNIFIED, emqx_route_unified). %% Mnesia table for message routing -define(ROUTING_NODE, emqx_routing_node). diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index a403efa84..d286af1d7 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -21,7 +21,6 @@ -include("emqx.hrl"). -include("logger.hrl"). -include("types.hrl"). --include_lib("mria/include/mria.hrl"). -include_lib("emqx/include/emqx_router.hrl"). %% Mnesia bootstrap @@ -73,11 +72,19 @@ code_change/3 ]). +%% test / debugging purposes +-export([is_unified_table_active/0]). + -type group() :: binary(). -type dest() :: node() | {group(), node()}. --dialyzer({nowarn_function, [cleanup_routes/1]}). +-record(routeidx, { + entry :: emqx_topic_index:key(dest()), + unused = [] :: nil() +}). + +-dialyzer({nowarn_function, [cleanup_routes_regular/1]}). %%-------------------------------------------------------------------- %% Mnesia bootstrap @@ -97,6 +104,19 @@ mnesia(boot) -> {write_concurrency, true} ]} ]} + ]), + ok = mria:create_table(?ROUTE_TAB_UNIFIED, [ + {type, ordered_set}, + {rlog_shard, ?ROUTE_SHARD}, + {storage, ram_copies}, + {record_name, routeidx}, + {attributes, record_info(fields, routeidx)}, + {storage_properties, [ + {ets, [ + {read_concurrency, true}, + {write_concurrency, auto} + ]} + ]} ]). %%-------------------------------------------------------------------- @@ -130,31 +150,54 @@ do_add_route(Topic) when is_binary(Topic) -> -spec do_add_route(emqx_types:topic(), dest()) -> ok | {error, term()}. do_add_route(Topic, Dest) when is_binary(Topic) -> - Route = #route{topic = Topic, dest = Dest}, - case lists:member(Route, lookup_routes(Topic)) of + case has_route(Topic, Dest) of true -> ok; false -> ok = emqx_router_helper:monitor(Dest), - case emqx_topic:wildcard(Topic) of - true -> - Fun = fun emqx_router_utils:insert_trie_route/2, - emqx_router_utils:maybe_trans(Fun, [?ROUTE_TAB, Route], ?ROUTE_SHARD); - false -> - emqx_router_utils:insert_direct_route(?ROUTE_TAB, Route) - end + mria_insert_route(is_unified_table_active(), Topic, Dest) end. +mria_insert_route(_Unified = true, Topic, Dest) -> + mria_insert_route_unified(Topic, Dest); +mria_insert_route(_Unified = false, Topic, Dest) -> + Route = #route{topic = Topic, dest = Dest}, + case emqx_topic:wildcard(Topic) of + true -> + mria_insert_route_update_trie(Route); + false -> + mria_insert_route(Route) + end. + +mria_insert_route_unified(Topic, Dest) -> + K = emqx_topic_index:make_key(Topic, Dest), + mria:dirty_write(?ROUTE_TAB_UNIFIED, #routeidx{entry = K}). + +mria_insert_route_update_trie(Route) -> + emqx_router_utils:maybe_trans( + fun emqx_router_utils:insert_trie_route/2, + [?ROUTE_TAB, Route], + ?ROUTE_SHARD + ). + +mria_insert_route(Route) -> + mria:dirty_write(?ROUTE_TAB, Route). + %% @doc Match routes -spec match_routes(emqx_types:topic()) -> [emqx_types:route()]. match_routes(Topic) when is_binary(Topic) -> - case match_trie(Topic) of - [] -> lookup_routes(Topic); - Matched -> lists:append([lookup_routes(To) || To <- [Topic | Matched]]) - end. + match_routes(is_unified_table_active(), Topic). -%% Optimize: routing table will be replicated to all router nodes. -match_trie(Topic) -> +match_routes(_Unified = true, Topic) -> + [match_to_route(M) || M <- match_unified(Topic)]; +match_routes(_Unified = false, Topic) -> + lookup_routes_regular(Topic) ++ + lists:flatmap(fun lookup_routes_regular/1, match_global_trie(Topic)). + +match_unified(Topic) -> + emqx_topic_index:matches(Topic, ?ROUTE_TAB_UNIFIED, []). + +match_global_trie(Topic) -> case emqx_trie:empty() of true -> []; false -> emqx_trie:match(Topic) @@ -162,12 +205,59 @@ match_trie(Topic) -> -spec lookup_routes(emqx_types:topic()) -> [emqx_types:route()]. lookup_routes(Topic) -> + case is_unified_table_active() of + true -> + lookup_routes_unified(Topic); + false -> + lookup_routes_regular(Topic) + end. + +lookup_routes_unified(Topic) -> + Pat = #routeidx{entry = emqx_topic_index:make_key(Topic, '$1')}, + [Dest || [Dest] <- ets:match(?ROUTE_TAB_UNIFIED, Pat)]. + +lookup_routes_regular(Topic) -> ets:lookup(?ROUTE_TAB, Topic). +match_to_route(M) -> + #route{topic = emqx_topic_index:get_topic(M), dest = emqx_topic_index:get_id(M)}. + -spec has_routes(emqx_types:topic()) -> boolean(). has_routes(Topic) when is_binary(Topic) -> + case is_unified_table_active() of + true -> + has_routes_unified(Topic); + false -> + has_routes_regular(Topic) + end. + +has_routes_unified(Topic) -> + Pat = #routeidx{entry = emqx_topic_index:mk_key(Topic, '$1'), _ = '_'}, + case ets:match(?ROUTE_TAB_UNIFIED, Pat, 1) of + {[_], _} -> + true; + _ -> + false + end. + +has_routes_regular(Topic) -> ets:member(?ROUTE_TAB, Topic). +-spec has_route(emqx_types:topic(), dest()) -> boolean(). +has_route(Topic, Dest) -> + case is_unified_table_active() of + true -> + has_route_unified(Topic, Dest); + false -> + has_route_regular(Topic, Dest) + end. + +has_route_unified(Topic, Dest) -> + ets:member(?ROUTE_TAB_UNIFIED, emqx_topic_index:make_key(Topic, Dest)). + +has_route_regular(Topic, Dest) -> + lists:any(fun(Route) -> Route#route.dest =:= Dest end, ets:lookup(?ROUTE_TAB, Topic)). + -spec delete_route(emqx_types:topic()) -> ok | {error, term()}. delete_route(Topic) when is_binary(Topic) -> delete_route(Topic, node()). @@ -182,17 +272,54 @@ do_delete_route(Topic) when is_binary(Topic) -> -spec do_delete_route(emqx_types:topic(), dest()) -> ok | {error, term()}. do_delete_route(Topic, Dest) -> + mria_delete_route(is_unified_table_active(), Topic, Dest). + +mria_delete_route(_Unified = true, Topic, Dest) -> + mria_delete_route_unified(Topic, Dest); +mria_delete_route(_Unified = false, Topic, Dest) -> Route = #route{topic = Topic, dest = Dest}, case emqx_topic:wildcard(Topic) of true -> - Fun = fun emqx_router_utils:delete_trie_route/2, - emqx_router_utils:maybe_trans(Fun, [?ROUTE_TAB, Route], ?ROUTE_SHARD); + mria_delete_route_update_trie(Route); false -> - emqx_router_utils:delete_direct_route(?ROUTE_TAB, Route) + mria_delete_route(Route) end. +mria_delete_route_unified(Topic, Dest) -> + K = emqx_topic_index:make_key(Topic, Dest), + mria:dirty_delete(?ROUTE_TAB_UNIFIED, K). + +mria_delete_route_update_trie(Route) -> + emqx_router_utils:maybe_trans( + fun emqx_router_utils:delete_trie_route/2, + [?ROUTE_TAB, Route], + ?ROUTE_SHARD + ). + +mria_delete_route(Route) -> + mria:dirty_delete_object(?ROUTE_TAB, Route). + +-spec is_unified_table_active() -> boolean(). +is_unified_table_active() -> + is_empty(?ROUTE_TAB) andalso + ((not is_empty(?ROUTE_TAB_UNIFIED)) orelse + emqx_config:get([broker, unified_routing_table])). + +is_empty(Tab) -> + % NOTE + % Supposedly, should be better than `ets:info(Tab, size)` because the latter suffers + % from `{decentralized_counters, true}` which is default when `write_concurrency` is + % either `auto` or `true`. + ets:first(Tab) =:= '$end_of_table'. + -spec topics() -> list(emqx_types:topic()). topics() -> + topics(is_unified_table_active()). + +topics(_Unified = true) -> + Pat = #routeidx{entry = '$1'}, + [emqx_topic_index:get_topic(K) || [K] <- ets:match(?ROUTE_TAB_UNIFIED, Pat)]; +topics(_Unified = false) -> mnesia:dirty_all_keys(?ROUTE_TAB). %% @doc Print routes to a topic @@ -207,23 +334,63 @@ print_routes(Topic) -> -spec cleanup_routes(node()) -> ok. cleanup_routes(Node) -> + case is_unified_table_active() of + true -> + cleanup_routes_unified(Node); + false -> + cleanup_routes_regular(Node) + end. + +cleanup_routes_unified(Node) -> + % NOTE + % No point in transaction here because all the operations on unified routing table + % are dirty. + ets:foldl( + fun(#routeidx{entry = K}, ok) -> + case emqx_topic_index:get_id(K) of + Node -> + mria:dirty_delete(?ROUTE_TAB_UNIFIED, K); + _ -> + ok + end + end, + ok, + ?ROUTE_TAB_UNIFIED + ). + +cleanup_routes_regular(Node) -> Patterns = [ #route{_ = '_', dest = Node}, #route{_ = '_', dest = {'_', Node}} ], - [ - mnesia:delete_object(?ROUTE_TAB, Route, write) - || Pat <- Patterns, - Route <- mnesia:match_object(?ROUTE_TAB, Pat, write) - ]. + mria:transaction(?ROUTE_SHARD, fun() -> + [ + mnesia:delete_object(?ROUTE_TAB, Route, write) + || Pat <- Patterns, + Route <- mnesia:match_object(?ROUTE_TAB, Pat, write) + ] + end). -spec foldl_routes(fun((emqx_types:route(), Acc) -> Acc), Acc) -> Acc. foldl_routes(FoldFun, AccIn) -> - ets:foldl(FoldFun, AccIn, ?ROUTE_TAB). + case is_unified_table_active() of + true -> + ets:foldl(mk_fold_fun_unified(FoldFun), AccIn, ?ROUTE_TAB_UNIFIED); + false -> + ets:foldl(FoldFun, AccIn, ?ROUTE_TAB) + end. -spec foldr_routes(fun((emqx_types:route(), Acc) -> Acc), Acc) -> Acc. foldr_routes(FoldFun, AccIn) -> - ets:foldr(FoldFun, AccIn, ?ROUTE_TAB). + case is_unified_table_active() of + true -> + ets:foldr(mk_fold_fun_unified(FoldFun), AccIn, ?ROUTE_TAB_UNIFIED); + false -> + ets:foldr(FoldFun, AccIn, ?ROUTE_TAB) + end. + +mk_fold_fun_unified(FoldFun) -> + fun(#routeidx{entry = K}, Acc) -> FoldFun(match_to_route(K), Acc) end. call(Router, Msg) -> gen_server:call(Router, Msg, infinity). diff --git a/apps/emqx/src/emqx_router_helper.erl b/apps/emqx/src/emqx_router_helper.erl index 61573fcff..77f1cd11b 100644 --- a/apps/emqx/src/emqx_router_helper.erl +++ b/apps/emqx/src/emqx_router_helper.erl @@ -148,11 +148,13 @@ handle_info({mnesia_table_event, Event}, State) -> handle_info({nodedown, Node}, State = #{nodes := Nodes}) -> case mria_rlog:role() of core -> + % TODO + % Node may flap, do we need to wait for any pending cleanups in `init/1` + % on the flapping node? + % This also implies changing lock id to `{?LOCK, Node}`. global:trans( {?LOCK, self()}, - fun() -> - mria:transaction(?ROUTE_SHARD, fun ?MODULE:cleanup_routes/1, [Node]) - end + fun() -> cleanup_routes(Node) end ), ok = mria:dirty_delete(?ROUTING_NODE, Node); replicant -> diff --git a/apps/emqx/src/emqx_topic_index.erl b/apps/emqx/src/emqx_topic_index.erl index 0b153ac01..5db8ce7cc 100644 --- a/apps/emqx/src/emqx_topic_index.erl +++ b/apps/emqx/src/emqx_topic_index.erl @@ -24,6 +24,8 @@ -export([match/2]). -export([matches/3]). +-export([make_key/2]). + -export([get_id/1]). -export([get_topic/1]). -export([get_record/2]). @@ -42,14 +44,18 @@ new() -> %% between regular and "materialized" indexes, for example. -spec insert(emqx_types:topic(), _ID, _Record, ets:table()) -> true. insert(Filter, ID, Record, Tab) -> - Key = key(Filter, ID), + Key = make_key(Filter, ID), true = ets:insert(Tab, {Key, Record}). %% @doc Delete an entry from the index that associates given topic filter to given %% record ID. Deleting non-existing entry is not an error. -spec delete(emqx_types:topic(), _ID, ets:table()) -> true. delete(Filter, ID, Tab) -> - true = ets:delete(Tab, key(Filter, ID)). + ets:delete(Tab, make_key(Filter, ID)). + +-spec make_key(emqx_types:topic(), ID) -> key(ID). +make_key(TopicOrFilter, ID) -> + emqx_trie_search:make_key(TopicOrFilter, ID). %% @doc Match given topic against the index and return the first match, or `false` if %% no match is found. @@ -84,8 +90,5 @@ get_record(K, Tab) -> [] end. -key(TopicOrFilter, ID) -> - emqx_trie_search:make_key(TopicOrFilter, ID). - make_nextf(Tab) -> fun(Key) -> ets:next(Tab, Key) end. diff --git a/apps/emqx/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl index 453f86257..04029e822 100644 --- a/apps/emqx/test/emqx_router_SUITE.erl +++ b/apps/emqx/test/emqx_router_SUITE.erl @@ -26,24 +26,37 @@ -define(R, emqx_router). -all() -> emqx_common_test_helpers:all(?MODULE). - -init_per_suite(Config) -> - PrevBootModules = application:get_env(emqx, boot_modules), - emqx_common_test_helpers:boot_modules([router]), - emqx_common_test_helpers:start_apps([]), +all() -> [ - {prev_boot_modules, PrevBootModules} - | Config + {group, routing_table_regular}, + {group, routing_table_unified} ]. -end_per_suite(Config) -> - PrevBootModules = ?config(prev_boot_modules, Config), - case PrevBootModules of - undefined -> ok; - {ok, Mods} -> emqx_common_test_helpers:boot_modules(Mods) - end, - emqx_common_test_helpers:stop_apps([]). +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + [ + {routing_table_regular, [], TCs}, + {routing_table_unified, [], TCs} + ]. + +init_per_group(GroupName, Config) -> + WorkDir = filename:join([?config(priv_dir, Config), GroupName]), + AppSpecs = [ + {emqx, #{ + config => mk_config(GroupName), + override_env => [{boot_modules, [router]}] + }} + ], + Apps = emqx_cth_suite:start(AppSpecs, #{work_dir => WorkDir}), + [{group_apps, Apps} | Config]. + +end_per_group(_GroupName, Config) -> + ok = emqx_cth_suite:stop(?config(group_apps, Config)). + +mk_config(routing_table_regular) -> + "broker.unified_routing_table = false"; +mk_config(routing_table_unified) -> + "broker.unified_routing_table = true". init_per_testcase(_TestCase, Config) -> clear_tables(), @@ -177,5 +190,5 @@ t_unexpected(_) -> clear_tables() -> lists:foreach( fun mnesia:clear_table/1, - [?ROUTE_TAB, ?TRIE, emqx_trie_node] + [?ROUTE_TAB, ?ROUTE_TAB_UNIFIED, ?TRIE] ). diff --git a/apps/emqx/test/emqx_router_helper_SUITE.erl b/apps/emqx/test/emqx_router_helper_SUITE.erl index c0796288e..12a3a34dd 100644 --- a/apps/emqx/test/emqx_router_helper_SUITE.erl +++ b/apps/emqx/test/emqx_router_helper_SUITE.erl @@ -26,32 +26,38 @@ -define(ROUTER_HELPER, emqx_router_helper). -all() -> emqx_common_test_helpers:all(?MODULE). +all() -> + [ + {group, routing_table_regular}, + {group, routing_table_unified} + ]. -init_per_suite(Config) -> - DistPid = - case net_kernel:nodename() of - ignored -> - %% calling `net_kernel:start' without `epmd' - %% running will result in a failure. - emqx_common_test_helpers:start_epmd(), - {ok, Pid} = net_kernel:start(['test@127.0.0.1', longnames]), - Pid; - _ -> - undefined - end, - emqx_common_test_helpers:start_apps([]), - [{dist_pid, DistPid} | Config]. +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + [ + {routing_table_regular, [], TCs}, + {routing_table_unified, [], TCs} + ]. -end_per_suite(Config) -> - DistPid = ?config(dist_pid, Config), - case DistPid of - Pid when is_pid(Pid) -> - net_kernel:stop(); - _ -> - ok - end, - emqx_common_test_helpers:stop_apps([]). +init_per_group(GroupName, Config) -> + WorkDir = filename:join([?config(priv_dir, Config), GroupName]), + AppSpecs = [{emqx, mk_config(GroupName)}], + Apps = emqx_cth_suite:start(AppSpecs, #{work_dir => WorkDir}), + [{group_name, GroupName}, {group_apps, Apps} | Config]. + +end_per_group(_GroupName, Config) -> + ok = emqx_cth_suite:stop(?config(group_apps, Config)). + +mk_config(routing_table_regular) -> + #{ + config => "broker.unified_routing_table = false", + override_env => [{boot_modules, [router]}] + }; +mk_config(routing_table_unified) -> + #{ + config => "broker.unified_routing_table = true", + override_env => [{boot_modules, [router]}] + }. init_per_testcase(TestCase, Config) when TestCase =:= t_cleanup_membership_mnesia_down; @@ -59,7 +65,16 @@ init_per_testcase(TestCase, Config) when TestCase =:= t_cleanup_monitor_node_down -> ok = snabbkaffe:start_trace(), - Slave = emqx_common_test_helpers:start_slave(some_node, []), + WorkDir = filename:join([?config(priv_dir, Config), ?config(group_name, Config), TestCase]), + [Slave] = emqx_cth_cluster:start( + [ + {?MODULE, #{ + apps => [{emqx, mk_config(?config(group_name, Config))}], + join_to => node() + }} + ], + #{work_dir => WorkDir} + ), [{slave, Slave} | Config]; init_per_testcase(_TestCase, Config) -> Config. @@ -70,9 +85,8 @@ end_per_testcase(TestCase, Config) when TestCase =:= t_cleanup_monitor_node_down -> Slave = ?config(slave, Config), - emqx_common_test_helpers:stop_slave(Slave), - mria:clear_table(?ROUTE_TAB), - snabbkaffe:stop(), + ok = emqx_cth_cluster:stop([Slave]), + ok = snabbkaffe:stop(), ok; end_per_testcase(_TestCase, _Config) -> ok. From 5d51687dbf3e554ac3125c3de5280d29ca4591ca Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 25 Aug 2023 14:49:50 +0400 Subject: [PATCH 080/119] feat(router): add unified routing table config option --- apps/emqx/src/emqx_schema.erl | 9 +++++++++ rel/i18n/emqx_schema.hocon | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index ff1446b6d..7268a7e59 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1358,6 +1358,15 @@ fields("broker") -> ref("broker_perf"), #{importance => ?IMPORTANCE_HIDDEN} )}, + {"unified_routing_table", + sc( + boolean(), + #{ + default => false, + importance => ?IMPORTANCE_HIDDEN, + desc => ?DESC(broker_unified_routing_table) + } + )}, %% FIXME: Need new design for shared subscription group {"shared_subscription_group", sc( diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 6c21b31ea..c63b248f4 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1549,6 +1549,12 @@ fields_ws_opts_max_frame_size.label: sys_event_messages.desc: """Client events messages.""" +broker_unified_routing_table.desc: +"""Enable unified routing table. +Unified routing table should increase both subscription and routing performance at the cost of slight increase in memory consumption per subscription. +NOTE: This is an experimental feature. +NOTE: Full non-rolling cluster restart is needed after enabling or disabling this option for it to take any effect.""" + broker_perf_trie_compaction.desc: """Enable trie path compaction. Enabling it significantly improves wildcard topic subscribe rate, if wildcard topics have unique prefixes like: 'sensor/{{id}}/+/', where ID is unique per subscriber. From e85789306b6b7677f06ea70e98ce42ec3ca2e37f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 25 Aug 2023 15:43:15 +0400 Subject: [PATCH 081/119] chore(router): drop test-only `has_routes/1` Seems that it's too much support work only for test purposes, where `lookup_routes/1` is nearly as usable. --- apps/emqx/src/emqx_router.erl | 23 +------------------ .../test/emqx_quic_multistreams_SUITE.erl | 2 +- apps/emqx/test/emqx_router_SUITE.erl | 4 ++-- apps/emqx/test/emqx_shared_sub_SUITE.erl | 4 ++-- .../emqx_bridge_kafka_impl_consumer_SUITE.erl | 2 +- 5 files changed, 7 insertions(+), 28 deletions(-) diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index d286af1d7..6859b98f3 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -50,7 +50,7 @@ -export([ match_routes/1, lookup_routes/1, - has_routes/1 + has_route/2 ]). -export([print_routes/1]). @@ -222,27 +222,6 @@ lookup_routes_regular(Topic) -> match_to_route(M) -> #route{topic = emqx_topic_index:get_topic(M), dest = emqx_topic_index:get_id(M)}. --spec has_routes(emqx_types:topic()) -> boolean(). -has_routes(Topic) when is_binary(Topic) -> - case is_unified_table_active() of - true -> - has_routes_unified(Topic); - false -> - has_routes_regular(Topic) - end. - -has_routes_unified(Topic) -> - Pat = #routeidx{entry = emqx_topic_index:mk_key(Topic, '$1'), _ = '_'}, - case ets:match(?ROUTE_TAB_UNIFIED, Pat, 1) of - {[_], _} -> - true; - _ -> - false - end. - -has_routes_regular(Topic) -> - ets:member(?ROUTE_TAB, Topic). - -spec has_route(emqx_types:topic(), dest()) -> boolean(). has_route(Topic, Dest) -> case is_unified_table_active() of diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index 1b45cb669..267782ff9 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -1094,7 +1094,7 @@ t_multi_streams_unsub(Config) -> ?retry( _Sleep2 = 100, _Attempts2 = 50, - false = emqx_router:has_routes(Topic) + [] = emqx_router:lookup_routes(Topic) ), case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of diff --git a/apps/emqx/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl index 04029e822..7dec039c8 100644 --- a/apps/emqx/test/emqx_router_SUITE.erl +++ b/apps/emqx/test/emqx_router_SUITE.erl @@ -176,9 +176,9 @@ t_print_routes(_) -> ?R:add_route(<<"+/+">>), ?R:print_routes(<<"a/b">>). -t_has_routes(_) -> +t_has_route(_) -> ?R:add_route(<<"devices/+/messages">>, node()), - ?assert(?R:has_routes(<<"devices/+/messages">>)), + ?assert(?R:has_route(<<"devices/+/messages">>, node())), ?R:delete_route(<<"devices/+/messages">>). t_unexpected(_) -> diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 6439981f6..7a7729878 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -1054,7 +1054,7 @@ t_queue_subscription(Config) when is_list(Config) -> begin ct:pal("routes: ~p", [ets:tab2list(emqx_route)]), %% FIXME: should ensure we have 2 subscriptions - true = emqx_router:has_routes(Topic) + [_] = emqx_router:lookup_routes(Topic) end ), @@ -1081,7 +1081,7 @@ t_queue_subscription(Config) when is_list(Config) -> %% _Attempts0 = 50, %% begin %% ct:pal("routes: ~p", [ets:tab2list(emqx_route)]), - %% false = emqx_router:has_routes(Topic) + %% [] = emqx_router:lookup_routes(Topic) %% end %% ), ct:sleep(500), diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 756fa8b3b..382e38fb7 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -1928,7 +1928,7 @@ t_node_joins_existing_cluster(Config) -> ?retry( _Sleep2 = 100, _Attempts2 = 50, - true = erpc:call(N2, emqx_router, has_routes, [MQTTTopic]) + [] =/= erpc:call(N2, emqx_router, lookup_routes, [MQTTTopic]) ), {ok, SRef1} = snabbkaffe:subscribe( From 8d2ebdea7e1810109c76a2f7d89ddd3297c3298a Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 25 Aug 2023 20:01:48 +0400 Subject: [PATCH 082/119] fix(router): generalize config option and make effects visible --- apps/emqx/src/emqx_app.erl | 4 +- apps/emqx/src/emqx_router.erl | 129 +++++++++++++------- apps/emqx/src/emqx_router_sup.erl | 2 + apps/emqx/src/emqx_schema.erl | 8 +- apps/emqx/test/emqx_router_SUITE.erl | 14 ++- apps/emqx/test/emqx_router_helper_SUITE.erl | 4 +- rel/i18n/emqx_schema.hocon | 4 +- 7 files changed, 111 insertions(+), 54 deletions(-) diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index 59a397836..4fae556ca 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -55,7 +55,9 @@ prep_stop(_State) -> emqx_boot:is_enabled(listeners) andalso emqx_listeners:stop(). -stop(_State) -> ok. +stop(_State) -> + ok = emqx_router:deinit_table_type(), + ok. -define(CONFIG_LOADER, config_loader). -define(DEFAULT_LOADER, emqx). diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 6859b98f3..7a8d88a55 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -72,8 +72,11 @@ code_change/3 ]). -%% test / debugging purposes --export([is_unified_table_active/0]). +-export([ + get_table_type/0, + init_table_type/0, + deinit_table_type/0 +]). -type group() :: binary(). @@ -155,12 +158,12 @@ do_add_route(Topic, Dest) when is_binary(Topic) -> ok; false -> ok = emqx_router_helper:monitor(Dest), - mria_insert_route(is_unified_table_active(), Topic, Dest) + mria_insert_route(get_table_type(), Topic, Dest) end. -mria_insert_route(_Unified = true, Topic, Dest) -> +mria_insert_route(unified, Topic, Dest) -> mria_insert_route_unified(Topic, Dest); -mria_insert_route(_Unified = false, Topic, Dest) -> +mria_insert_route(regular, Topic, Dest) -> Route = #route{topic = Topic, dest = Dest}, case emqx_topic:wildcard(Topic) of true -> @@ -186,11 +189,11 @@ mria_insert_route(Route) -> %% @doc Match routes -spec match_routes(emqx_types:topic()) -> [emqx_types:route()]. match_routes(Topic) when is_binary(Topic) -> - match_routes(is_unified_table_active(), Topic). + match_routes(get_table_type(), Topic). -match_routes(_Unified = true, Topic) -> +match_routes(unified, Topic) -> [match_to_route(M) || M <- match_unified(Topic)]; -match_routes(_Unified = false, Topic) -> +match_routes(regular, Topic) -> lookup_routes_regular(Topic) ++ lists:flatmap(fun lookup_routes_regular/1, match_global_trie(Topic)). @@ -205,10 +208,10 @@ match_global_trie(Topic) -> -spec lookup_routes(emqx_types:topic()) -> [emqx_types:route()]. lookup_routes(Topic) -> - case is_unified_table_active() of - true -> + case get_table_type() of + unified -> lookup_routes_unified(Topic); - false -> + regular -> lookup_routes_regular(Topic) end. @@ -224,10 +227,10 @@ match_to_route(M) -> -spec has_route(emqx_types:topic(), dest()) -> boolean(). has_route(Topic, Dest) -> - case is_unified_table_active() of - true -> + case get_table_type() of + unified -> has_route_unified(Topic, Dest); - false -> + regular -> has_route_regular(Topic, Dest) end. @@ -251,11 +254,11 @@ do_delete_route(Topic) when is_binary(Topic) -> -spec do_delete_route(emqx_types:topic(), dest()) -> ok | {error, term()}. do_delete_route(Topic, Dest) -> - mria_delete_route(is_unified_table_active(), Topic, Dest). + mria_delete_route(get_table_type(), Topic, Dest). -mria_delete_route(_Unified = true, Topic, Dest) -> +mria_delete_route(unified, Topic, Dest) -> mria_delete_route_unified(Topic, Dest); -mria_delete_route(_Unified = false, Topic, Dest) -> +mria_delete_route(regular, Topic, Dest) -> Route = #route{topic = Topic, dest = Dest}, case emqx_topic:wildcard(Topic) of true -> @@ -278,27 +281,14 @@ mria_delete_route_update_trie(Route) -> mria_delete_route(Route) -> mria:dirty_delete_object(?ROUTE_TAB, Route). --spec is_unified_table_active() -> boolean(). -is_unified_table_active() -> - is_empty(?ROUTE_TAB) andalso - ((not is_empty(?ROUTE_TAB_UNIFIED)) orelse - emqx_config:get([broker, unified_routing_table])). - -is_empty(Tab) -> - % NOTE - % Supposedly, should be better than `ets:info(Tab, size)` because the latter suffers - % from `{decentralized_counters, true}` which is default when `write_concurrency` is - % either `auto` or `true`. - ets:first(Tab) =:= '$end_of_table'. - -spec topics() -> list(emqx_types:topic()). topics() -> - topics(is_unified_table_active()). + topics(get_table_type()). -topics(_Unified = true) -> +topics(unified) -> Pat = #routeidx{entry = '$1'}, [emqx_topic_index:get_topic(K) || [K] <- ets:match(?ROUTE_TAB_UNIFIED, Pat)]; -topics(_Unified = false) -> +topics(regular) -> mnesia:dirty_all_keys(?ROUTE_TAB). %% @doc Print routes to a topic @@ -313,10 +303,10 @@ print_routes(Topic) -> -spec cleanup_routes(node()) -> ok. cleanup_routes(Node) -> - case is_unified_table_active() of - true -> + case get_table_type() of + unified -> cleanup_routes_unified(Node); - false -> + regular -> cleanup_routes_regular(Node) end. @@ -352,19 +342,19 @@ cleanup_routes_regular(Node) -> -spec foldl_routes(fun((emqx_types:route(), Acc) -> Acc), Acc) -> Acc. foldl_routes(FoldFun, AccIn) -> - case is_unified_table_active() of - true -> + case get_table_type() of + unified -> ets:foldl(mk_fold_fun_unified(FoldFun), AccIn, ?ROUTE_TAB_UNIFIED); - false -> + regular -> ets:foldl(FoldFun, AccIn, ?ROUTE_TAB) end. -spec foldr_routes(fun((emqx_types:route(), Acc) -> Acc), Acc) -> Acc. foldr_routes(FoldFun, AccIn) -> - case is_unified_table_active() of - true -> + case get_table_type() of + unified -> ets:foldr(mk_fold_fun_unified(FoldFun), AccIn, ?ROUTE_TAB_UNIFIED); - false -> + regular -> ets:foldr(FoldFun, AccIn, ?ROUTE_TAB) end. @@ -377,6 +367,61 @@ call(Router, Msg) -> pick(Topic) -> gproc_pool:pick_worker(router_pool, Topic). +%%-------------------------------------------------------------------- +%% Routing table type +%% -------------------------------------------------------------------- + +-define(PT_TABLE_TYPE, {?MODULE, tabtype}). + +-type tabtype() :: regular | unified. + +-spec get_table_type() -> tabtype(). +get_table_type() -> + persistent_term:get(?PT_TABLE_TYPE). + +-spec init_table_type() -> ok. +init_table_type() -> + ConfType = emqx_config:get([broker, routing_table_type]), + Type = choose_table_type(ConfType), + ok = persistent_term:put(?PT_TABLE_TYPE, Type), + case Type of + ConfType -> + ?SLOG(info, #{ + msg => "routing_table_type_used", + type => Type + }); + _ -> + ?SLOG(notice, #{ + msg => "configured_routing_table_type_unacceptable", + type => Type, + configured => ConfType, + reason => + "Could not use configured routing table type because " + "there's already non-empty routing table of another type." + }) + end. + +-spec deinit_table_type() -> ok. +deinit_table_type() -> + _ = persistent_term:erase(?PT_TABLE_TYPE), + ok. + +-spec choose_table_type(tabtype()) -> tabtype(). +choose_table_type(ConfType) -> + IsEmptyRegular = is_empty(?ROUTE_TAB), + IsEmptyUnified = is_empty(?ROUTE_TAB_UNIFIED), + case {IsEmptyRegular, IsEmptyUnified} of + {true, true} -> + ConfType; + {false, true} -> + regular; + {true, false} -> + unified + end. + +is_empty(Tab) -> + ets:first(Tab) =:= '$end_of_table'. + %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_router_sup.erl b/apps/emqx/src/emqx_router_sup.erl index 0fa48d9d2..398edc321 100644 --- a/apps/emqx/src/emqx_router_sup.erl +++ b/apps/emqx/src/emqx_router_sup.erl @@ -23,6 +23,8 @@ -export([init/1]). start_link() -> + %% Init and log routing table type + ok = emqx_router:init_table_type(), supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 7268a7e59..818688d86 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1358,13 +1358,13 @@ fields("broker") -> ref("broker_perf"), #{importance => ?IMPORTANCE_HIDDEN} )}, - {"unified_routing_table", + {"routing_table_type", sc( - boolean(), + hoconsc:enum([regular, unified]), #{ - default => false, + default => regular, importance => ?IMPORTANCE_HIDDEN, - desc => ?DESC(broker_unified_routing_table) + desc => ?DESC(broker_routing_table_type) } )}, %% FIXME: Need new design for shared subscription group diff --git a/apps/emqx/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl index 7dec039c8..f9e1c6998 100644 --- a/apps/emqx/test/emqx_router_SUITE.erl +++ b/apps/emqx/test/emqx_router_SUITE.erl @@ -48,15 +48,15 @@ init_per_group(GroupName, Config) -> }} ], Apps = emqx_cth_suite:start(AppSpecs, #{work_dir => WorkDir}), - [{group_apps, Apps} | Config]. + [{group_apps, Apps}, {group_name, GroupName} | Config]. end_per_group(_GroupName, Config) -> ok = emqx_cth_suite:stop(?config(group_apps, Config)). mk_config(routing_table_regular) -> - "broker.unified_routing_table = false"; + "broker.routing_table_type = regular"; mk_config(routing_table_unified) -> - "broker.unified_routing_table = true". + "broker.routing_table_type = unified". init_per_testcase(_TestCase, Config) -> clear_tables(), @@ -83,6 +83,14 @@ end_per_testcase(_TestCase, _Config) -> % t_topics(_) -> % error('TODO'). +t_verify_type(Config) -> + case ?config(group_name, Config) of + routing_table_regular -> + ?assertEqual(regular, ?R:get_table_type()); + routing_table_unified -> + ?assertEqual(unified, ?R:get_table_type()) + end. + t_add_delete(_) -> ?R:add_route(<<"a/b/c">>), ?R:add_route(<<"a/b/c">>, node()), diff --git a/apps/emqx/test/emqx_router_helper_SUITE.erl b/apps/emqx/test/emqx_router_helper_SUITE.erl index 12a3a34dd..26b9d8ddd 100644 --- a/apps/emqx/test/emqx_router_helper_SUITE.erl +++ b/apps/emqx/test/emqx_router_helper_SUITE.erl @@ -50,12 +50,12 @@ end_per_group(_GroupName, Config) -> mk_config(routing_table_regular) -> #{ - config => "broker.unified_routing_table = false", + config => "broker.routing_table_type = regular", override_env => [{boot_modules, [router]}] }; mk_config(routing_table_unified) -> #{ - config => "broker.unified_routing_table = true", + config => "broker.routing_table_type = unified", override_env => [{boot_modules, [router]}] }. diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index c63b248f4..d90fab47b 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1549,8 +1549,8 @@ fields_ws_opts_max_frame_size.label: sys_event_messages.desc: """Client events messages.""" -broker_unified_routing_table.desc: -"""Enable unified routing table. +broker_routing_table_type.desc: +"""Routing table type. Unified routing table should increase both subscription and routing performance at the cost of slight increase in memory consumption per subscription. NOTE: This is an experimental feature. NOTE: Full non-rolling cluster restart is needed after enabling or disabling this option for it to take any effect.""" From 2d931a05121c9bfd710c2d3832735154f5f4ab55 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 8 Aug 2023 20:06:21 +0400 Subject: [PATCH 083/119] test(evict): do not disable router module in cluster nodes --- .../test/emqx_eviction_agent_test_helpers.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl index 7425cb145..17c291eb6 100644 --- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl +++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl @@ -81,7 +81,7 @@ start_cluster(NamesWithPorts, Apps, Env) -> NamesWithPorts ), Opts0 = [ - {env, [{emqx, boot_modules, [broker, listeners]}] ++ Env}, + {env, Env}, {apps, Apps}, {conf, [{[listeners, Proto, default, enable], false} || Proto <- [ssl, ws, wss]] ++ From 0b86f04bae35b76d78bc61e46c82938db083db44 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Wed, 23 Aug 2023 23:27:04 -0300 Subject: [PATCH 084/119] fix(kafka): fix template processing for header Fixes https://emqx.atlassian.net/browse/EMQX-10846 --- .../src/emqx_bridge_kafka_impl_producer.erl | 5 +- .../emqx_bridge_kafka_impl_producer_SUITE.erl | 121 ++++++++++++------ changes/ee/fix-11527.en.md | 1 + 3 files changed, 88 insertions(+), 39 deletions(-) create mode 100644 changes/ee/fix-11527.en.md diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 16e9561e8..0b8f526f6 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -664,6 +664,9 @@ kvlist_headers([#{<<"key">> := K, <<"value">> := V} | Headers], Acc) -> kvlist_headers(Headers, [{K, V} | Acc]); kvlist_headers([{K, V} | Headers], Acc) -> kvlist_headers(Headers, [{K, V} | Acc]); +kvlist_headers([KVList | Headers], Acc) when is_list(KVList) -> + %% for instance, when user sets a json list as headers like '[{"foo":"bar"}, {"foo2":"bar2"}]'. + kvlist_headers(KVList ++ Headers, Acc); kvlist_headers([BadHeader | _], _) -> throw({bad_kafka_header, BadHeader}). @@ -694,7 +697,7 @@ merge_kafka_headers(HeadersTks, ExtHeaders, Msg) -> [undefined] -> ExtHeaders; [MaybeJson] when is_binary(MaybeJson) -> - case emqx_utils_json:safe_decode(MaybeJson) of + case emqx_utils_json:safe_decode(MaybeJson, [return_maps]) of {ok, JsonTerm} when is_map(JsonTerm) -> maps:to_list(JsonTerm) ++ ExtHeaders; {ok, JsonTerm} when is_list(JsonTerm) -> diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index 00fa21626..ba11ddf14 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -560,7 +560,7 @@ t_send_message_with_headers(Config) -> "kafka_hosts_string" => HostsString, "kafka_topic" => KafkaTopic, "instance_id" => ResourceId, - "kafka_headers" => <<"${pub_props}">>, + "kafka_headers" => <<"${payload.header}">>, "kafka_ext_headers" => emqx_utils_json:encode( [ #{ @@ -568,8 +568,8 @@ t_send_message_with_headers(Config) -> <<"kafka_ext_header_value">> => <<"${clientid}">> }, #{ - <<"kafka_ext_header_key">> => <<"payload">>, - <<"kafka_ext_header_value">> => <<"${payload}">> + <<"kafka_ext_header_key">> => <<"ext_header_val">>, + <<"kafka_ext_header_value">> => <<"${payload.ext_header_val}">> } ] ), @@ -587,12 +587,42 @@ t_send_message_with_headers(Config) -> ), ConfigAtom = ConfigAtom1#{bridge_name => Name}, {ok, State} = ?PRODUCER:on_start(ResourceId, ConfigAtom), - Time = erlang:unique_integer(), - BinTime = integer_to_binary(Time), - Msg = #{ - clientid => BinTime, - payload => <<"payload">>, - timestamp => Time + Time1 = erlang:unique_integer(), + BinTime1 = integer_to_binary(Time1), + Payload1 = emqx_utils_json:encode( + #{ + <<"header">> => #{ + <<"foo">> => <<"bar">> + }, + <<"ext_header_val">> => <<"ext header ok">> + } + ), + Msg1 = #{ + clientid => BinTime1, + payload => Payload1, + timestamp => Time1 + }, + Time2 = erlang:unique_integer(), + BinTime2 = integer_to_binary(Time2), + Payload2 = emqx_utils_json:encode( + #{ + <<"header">> => [ + #{ + <<"key">> => <<"foo1">>, + <<"value">> => <<"bar1">> + }, + #{ + <<"key">> => <<"foo2">>, + <<"value">> => <<"bar2">> + } + ], + <<"ext_header_val">> => <<"ext header ok">> + } + ), + Msg2 = #{ + clientid => BinTime2, + payload => Payload2, + timestamp => Time2 }, {ok, Offset} = resolve_kafka_offset(kafka_hosts(), KafkaTopic, 0), ct:pal("base offset before testing ~p", [Offset]), @@ -603,7 +633,8 @@ t_send_message_with_headers(Config) -> end, ?check_trace( begin - ok = send(Config, ResourceId, Msg, State) + ok = send(Config, ResourceId, Msg1, State), + ok = send(Config, ResourceId, Msg2, State) end, fun(Trace) -> ?assertMatch( @@ -616,11 +647,27 @@ t_send_message_with_headers(Config) -> [{var, [<<"clientid">>]}] }, { - [{str, <<"payload">>}], - [{var, [<<"payload">>]}] + [{str, <<"ext_header_val">>}], + [{var, [<<"payload">>, <<"ext_header_val">>]}] } ], - headers_tokens := [{var, [<<"pub_props">>]}], + headers_tokens := [{var, [<<"payload">>, <<"header">>]}], + headers_val_encode_mode := json + } + }, + #{ + headers_config := #{ + ext_headers_tokens := [ + { + [{str, <<"clientid">>}], + [{var, [<<"clientid">>]}] + }, + { + [{str, <<"ext_header_val">>}], + [{var, [<<"payload">>, <<"ext_header_val">>]}] + } + ], + headers_tokens := [{var, [<<"payload">>, <<"header">>]}], headers_val_encode_mode := json } } @@ -629,16 +676,28 @@ t_send_message_with_headers(Config) -> ) end ), - {ok, {_, [KafkaMsg]}} = brod:fetch(kafka_hosts(), KafkaTopic, 0, Offset), + {ok, {_, KafkaMsgs}} = brod:fetch(kafka_hosts(), KafkaTopic, 0, Offset), ?assertMatch( - #kafka_message{ - headers = [ - {<<"clientid">>, _}, - {<<"payload">>, <<"\"payload\"">>} - ], - key = BinTime - }, - KafkaMsg + [ + #kafka_message{ + headers = [ + {<<"foo">>, <<"\"bar\"">>}, + {<<"clientid">>, _}, + {<<"ext_header_val">>, <<"\"ext header ok\"">>} + ], + key = BinTime1 + }, + #kafka_message{ + headers = [ + {<<"foo1">>, <<"\"bar1\"">>}, + {<<"foo2">>, <<"\"bar2\"">>}, + {<<"clientid">>, _}, + {<<"ext_header_val">>, <<"\"ext header ok\"">>} + ], + key = BinTime2 + } + ], + KafkaMsgs ), %% TODO: refactor those into init/end per testcase ok = ?PRODUCER:on_stop(ResourceId, State), @@ -769,7 +828,7 @@ t_wrong_headers_from_message(Config) -> timestamp => Time2 }, ?assertError( - {badmatch, {error, {unrecoverable_error, {bad_kafka_header, [{<<"foo">>, <<"bar">>}]}}}}, + {badmatch, {error, {unrecoverable_error, {bad_kafka_header, #{<<"foo">> := <<"bar">>}}}}}, send(Config, ResourceId, Msg2, State) ), Time3 = erlang:unique_integer(), @@ -780,23 +839,9 @@ t_wrong_headers_from_message(Config) -> timestamp => Time3 }, ?assertError( - {badmatch, {error, {unrecoverable_error, {bad_kafka_header, [{<<"key">>, <<"foo">>}]}}}}, + {badmatch, {error, {unrecoverable_error, {bad_kafka_header, #{<<"key">> := <<"foo">>}}}}}, send(Config, ResourceId, Msg3, State) ), - Time4 = erlang:unique_integer(), - Payload4 = <<"[{\"key\":\"foo\", \"value\":\"bar\"}]">>, - Msg4 = #{ - clientid => integer_to_binary(Time4), - payload => Payload4, - timestamp => Time4 - }, - ?assertError( - {badmatch, - {error, - {unrecoverable_error, - {bad_kafka_header, [{<<"key">>, <<"foo">>}, {<<"value">>, <<"bar">>}]}}}}, - send(Config, ResourceId, Msg4, State) - ), %% TODO: refactor those into init/end per testcase ok = ?PRODUCER:on_stop(ResourceId, State), ?assertEqual([], supervisor:which_children(wolff_client_sup)), diff --git a/changes/ee/fix-11527.en.md b/changes/ee/fix-11527.en.md new file mode 100644 index 000000000..33d077e94 --- /dev/null +++ b/changes/ee/fix-11527.en.md @@ -0,0 +1 @@ +Fixed an issue with Kafka header handling when placeholders resolve to an array of key-value pairs (e.g.: `[{"key": "foo", "value": "bar"}]`). From cc3ba18734072e29536305b562a920d0a3f5788a Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Mon, 28 Aug 2023 19:37:34 -0300 Subject: [PATCH 085/119] fix: increment dropped message counter when bridge is unhealthy Fixes https://emqx.atlassian.net/browse/EMQX-10767 --- apps/emqx_resource/src/emqx_resource.erl | 2 ++ .../test/emqx_resource_SUITE.erl | 23 +++++++++++++++++++ changes/ce/fix-11534.en.md | 1 + 3 files changed, 26 insertions(+) create mode 100644 changes/ce/fix-11534.en.md diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index d0d93a701..1e0f0b45d 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -281,8 +281,10 @@ query(ResId, Request, Opts) -> {ok, _Group, #{query_mode := QM, error := Error}} -> case {QM, Error} of {_, unhealthy_target} -> + emqx_resource_metrics:dropped_resource_stopped_inc(ResId), ?RESOURCE_ERROR(unhealthy_target, "unhealthy target"); {_, {unhealthy_target, _Message}} -> + emqx_resource_metrics:dropped_resource_stopped_inc(ResId), ?RESOURCE_ERROR(unhealthy_target, "unhealthy target"); {simple_async, _} -> %% TODO(5.1.1): pass Resource instead of ResId to simple APIs diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 77e680933..d4bc9a8a7 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -899,6 +899,29 @@ t_healthy(_) -> end ). +t_unhealthy_target(_) -> + HealthCheckError = {unhealthy_target, "some message"}, + ?assertMatch( + {ok, _}, + emqx_resource:create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource, health_check_error => {msg, HealthCheckError}} + ) + ), + ?assertEqual( + {ok, disconnected}, + emqx_resource:health_check(?ID) + ), + ?assertMatch( + {ok, _Group, #{error := HealthCheckError}}, + emqx_resource_manager:lookup(?ID) + ), + %% messages are dropped when bridge is unhealthy + emqx_resource:query(?ID, message), + ?assertEqual(1, emqx_resource_metrics:dropped_resource_stopped_get(?ID)). + t_stop_start(_) -> ?check_trace( begin diff --git a/changes/ce/fix-11534.en.md b/changes/ce/fix-11534.en.md new file mode 100644 index 000000000..15c89f392 --- /dev/null +++ b/changes/ce/fix-11534.en.md @@ -0,0 +1 @@ +Fixed increment on data bridge statistics when bridge is unhealthy. Now, messages sent to unhealthy bridges are being counted as dropped messages. From 4e7ba5d35ff128898f720ac8b6b1ea8102f885a0 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 29 Aug 2023 10:32:59 +0800 Subject: [PATCH 086/119] chore: ciphers should allow empty space(comma_separated_binary) --- apps/emqx/src/emqx_connection.erl | 6 +-- apps/emqx/src/emqx_schema.erl | 5 +- .../emqx/test/emqx_listeners_update_SUITE.erl | 54 ++++++++++++++++++- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 60f38e2bd..db36fbea9 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -1216,9 +1216,9 @@ inc_counter(Key, Inc) -> set_tcp_keepalive({quic, _Listener}) -> ok; set_tcp_keepalive({Type, Id}) -> - Conf = emqx_config:get_listener_conf(Type, Id, [tcp_options, keepalive], <<"none">>), - case iolist_to_binary(Conf) of - <<"none">> -> + Conf = emqx_config:get_listener_conf(Type, Id, [tcp_options, keepalive], "none"), + case Conf of + "none" -> ok; Value -> %% the value is already validated by schema, so we do not validate it again. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index ff1446b6d..db2c0b4a8 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2331,7 +2331,8 @@ converter_ciphers(<<>>, _Opts) -> []; converter_ciphers(Ciphers, _Opts) when is_list(Ciphers) -> Ciphers; converter_ciphers(Ciphers, _Opts) when is_binary(Ciphers) -> - binary:split(Ciphers, <<",">>, [global]). + {ok, List} = to_comma_separated_binary(binary_to_list(Ciphers)), + List. default_ciphers(Which) -> lists:map( @@ -2649,7 +2650,7 @@ validate_tcp_keepalive(Value) -> %% @doc This function is used as value validator and also run-time parser. parse_tcp_keepalive(Str) -> try - [Idle, Interval, Probes] = binary:split(iolist_to_binary(Str), <<",">>, [global]), + {ok, [Idle, Interval, Probes]} = to_comma_separated_binary(Str), %% use 10 times the Linux defaults as range limit IdleInt = parse_ka_int(Idle, "Idle", 1, 7200_0), IntervalInt = parse_ka_int(Interval, "Interval", 1, 75_0), diff --git a/apps/emqx/test/emqx_listeners_update_SUITE.erl b/apps/emqx/test/emqx_listeners_update_SUITE.erl index 5c51d990a..c1fdff2e2 100644 --- a/apps/emqx/test/emqx_listeners_update_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_update_SUITE.erl @@ -116,6 +116,55 @@ t_update_conf(_Conf) -> ?assert(is_running('wss:default')), ok. +t_update_tcp_keepalive_conf(_Conf) -> + Keepalive = <<"240,30,5">>, + KeepaliveStr = binary_to_list(Keepalive), + Raw = emqx:get_raw_config(?LISTENERS), + Raw1 = emqx_utils_maps:deep_put( + [<<"tcp">>, <<"default">>, <<"bind">>], Raw, <<"127.0.0.1:1883">> + ), + Raw2 = emqx_utils_maps:deep_put( + [<<"tcp">>, <<"default">>, <<"tcp_options">>, <<"keepalive">>], Raw1, Keepalive + ), + ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw2)), + ?assertMatch( + #{ + <<"tcp">> := #{ + <<"default">> := #{ + <<"bind">> := <<"127.0.0.1:1883">>, + <<"tcp_options">> := #{<<"keepalive">> := Keepalive} + } + } + }, + emqx:get_raw_config(?LISTENERS) + ), + ?assertMatch( + #{tcp := #{default := #{tcp_options := #{keepalive := KeepaliveStr}}}}, + emqx:get_config(?LISTENERS) + ), + Keepalive2 = <<" 241, 31, 6 ">>, + KeepaliveStr2 = binary_to_list(Keepalive2), + Raw3 = emqx_utils_maps:deep_put( + [<<"tcp">>, <<"default">>, <<"tcp_options">>, <<"keepalive">>], Raw1, Keepalive2 + ), + ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw3)), + ?assertMatch( + #{ + <<"tcp">> := #{ + <<"default">> := #{ + <<"bind">> := <<"127.0.0.1:1883">>, + <<"tcp_options">> := #{<<"keepalive">> := Keepalive2} + } + } + }, + emqx:get_raw_config(?LISTENERS) + ), + ?assertMatch( + #{tcp := #{default := #{tcp_options := #{keepalive := KeepaliveStr2}}}}, + emqx:get_config(?LISTENERS) + ), + ok. + t_update_empty_ssl_options_conf(_Conf) -> Raw = emqx:get_raw_config(?LISTENERS), Raw1 = emqx_utils_maps:deep_put( @@ -139,10 +188,11 @@ t_update_empty_ssl_options_conf(_Conf) -> Raw7 = emqx_utils_maps:deep_put( [<<"wss">>, <<"default">>, <<"ssl_options">>, <<"ciphers">>], Raw6, <<"">> ), + Ciphers = <<"TLS_AES_256_GCM_SHA384, TLS_AES_128_GCM_SHA256 ">>, Raw8 = emqx_utils_maps:deep_put( [<<"ssl">>, <<"default">>, <<"ssl_options">>, <<"ciphers">>], Raw7, - <<"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256">> + Ciphers ), ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw8)), ?assertMatch( @@ -153,7 +203,7 @@ t_update_empty_ssl_options_conf(_Conf) -> <<"bind">> := <<"127.0.0.1:8883">>, <<"ssl_options">> := #{ <<"cacertfile">> := <<"">>, - <<"ciphers">> := <<"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256">> + <<"ciphers">> := Ciphers } } }, From 270fd107b296700f38c6600f27a71e661633d8a1 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 29 Aug 2023 17:31:26 +0400 Subject: [PATCH 087/119] feat(triesearch): allow reusing filter tokenization result As a kind of micro-optimization. --- apps/emqx/src/emqx_topic_gbt.erl | 8 +++--- apps/emqx/src/emqx_topic_index.erl | 7 ++--- apps/emqx/src/emqx_trie_search.erl | 23 ++++++++++++----- apps/emqx/test/emqx_topic_index_SUITE.erl | 11 ++++++++ apps/emqx/test/emqx_trie_search_tests.erl | 31 +++++++++++++++++------ 5 files changed, 58 insertions(+), 22 deletions(-) diff --git a/apps/emqx/src/emqx_topic_gbt.erl b/apps/emqx/src/emqx_topic_gbt.erl index dade70cee..063cba21d 100644 --- a/apps/emqx/src/emqx_topic_gbt.erl +++ b/apps/emqx/src/emqx_topic_gbt.erl @@ -29,8 +29,8 @@ -export([get_topic/1]). -export([get_record/2]). --type word() :: binary() | '+' | '#'. --type key(ID) :: {[word()], {ID}}. +-type key(ID) :: emqx_trie_search:key(ID). +-type words() :: emqx_trie_search:words(). -type match(ID) :: key(ID). -type name() :: any(). @@ -50,7 +50,7 @@ new(Name) -> %% @doc Insert a new entry into the index that associates given topic filter to given %% record ID, and attaches arbitrary record to the entry. This allows users to choose %% between regular and "materialized" indexes, for example. --spec insert(emqx_types:topic(), _ID, _Record, name()) -> true. +-spec insert(emqx_types:topic() | words(), _ID, _Record, name()) -> true. insert(Filter, ID, Record, Name) -> Tree = gbt(Name), Key = key(Filter, ID), @@ -59,7 +59,7 @@ insert(Filter, ID, Record, Name) -> %% @doc Delete an entry from the index that associates given topic filter to given %% record ID. Deleting non-existing entry is not an error. --spec delete(emqx_types:topic(), _ID, name()) -> true. +-spec delete(emqx_types:topic() | words(), _ID, name()) -> true. delete(Filter, ID, Name) -> Tree = gbt(Name), Key = key(Filter, ID), diff --git a/apps/emqx/src/emqx_topic_index.erl b/apps/emqx/src/emqx_topic_index.erl index 5db8ce7cc..59dfdfeab 100644 --- a/apps/emqx/src/emqx_topic_index.erl +++ b/apps/emqx/src/emqx_topic_index.erl @@ -32,6 +32,7 @@ -type key(ID) :: emqx_trie_search:key(ID). -type match(ID) :: key(ID). +-type words() :: emqx_trie_search:words(). %% @doc Create a new ETS table suitable for topic index. %% Usable mostly for testing purposes. @@ -42,18 +43,18 @@ new() -> %% @doc Insert a new entry into the index that associates given topic filter to given %% record ID, and attaches arbitrary record to the entry. This allows users to choose %% between regular and "materialized" indexes, for example. --spec insert(emqx_types:topic(), _ID, _Record, ets:table()) -> true. +-spec insert(emqx_types:topic() | words(), _ID, _Record, ets:table()) -> true. insert(Filter, ID, Record, Tab) -> Key = make_key(Filter, ID), true = ets:insert(Tab, {Key, Record}). %% @doc Delete an entry from the index that associates given topic filter to given %% record ID. Deleting non-existing entry is not an error. --spec delete(emqx_types:topic(), _ID, ets:table()) -> true. +-spec delete(emqx_types:topic() | words(), _ID, ets:table()) -> true. delete(Filter, ID, Tab) -> ets:delete(Tab, make_key(Filter, ID)). --spec make_key(emqx_types:topic(), ID) -> key(ID). +-spec make_key(emqx_types:topic() | words(), ID) -> key(ID). make_key(TopicOrFilter, ID) -> emqx_trie_search:make_key(TopicOrFilter, ID). diff --git a/apps/emqx/src/emqx_trie_search.erl b/apps/emqx/src/emqx_trie_search.erl index b774e1459..c8c088b58 100644 --- a/apps/emqx/src/emqx_trie_search.erl +++ b/apps/emqx/src/emqx_trie_search.erl @@ -98,24 +98,24 @@ -module(emqx_trie_search). --export([make_key/2]). +-export([make_key/2, filter/1]). -export([match/2, matches/3, get_id/1, get_topic/1]). --export_type([key/1, word/0, nextf/0, opts/0]). +-export_type([key/1, word/0, words/0, nextf/0, opts/0]). -define(END, '$end_of_table'). -type word() :: binary() | '+' | '#'. +-type words() :: [word()]. -type base_key() :: {binary() | [word()], {}}. -type key(ID) :: {binary() | [word()], {ID}}. -type nextf() :: fun((key(_) | base_key()) -> ?END | key(_)). -type opts() :: [unique | return_first]. %% @doc Make a search-key for the given topic. --spec make_key(emqx_types:topic(), ID) -> key(ID). +-spec make_key(emqx_types:topic() | words(), ID) -> key(ID). make_key(Topic, ID) when is_binary(Topic) -> - Words = filter_words(Topic), - case emqx_topic:wildcard(Words) of - true -> + case filter(Topic) of + Words when is_list(Words) -> %% it's a wildcard {Words, {ID}}; false -> @@ -123,7 +123,15 @@ make_key(Topic, ID) when is_binary(Topic) -> %% because they can be found with direct lookups. %% it is also more compact in memory. {Topic, {ID}} - end. + end; +make_key(Words, ID) when is_list(Words) -> + {Words, {ID}}. + +%% @doc Parse a topic filter into a list of words. Returns `false` if it's not a filter. +-spec filter(emqx_types:topic()) -> words() | false. +filter(Topic) -> + Words = filter_words(Topic), + emqx_topic:wildcard(Words) andalso Words. %% @doc Extract record ID from the match. -spec get_id(key(ID)) -> ID. @@ -325,6 +333,7 @@ filter_words(Topic) when is_binary(Topic) -> % `match_filter/3` expects. [word(W, filter) || W <- emqx_topic:tokens(Topic)]. +-spec topic_words(emqx_types:topic()) -> [binary()]. topic_words(Topic) when is_binary(Topic) -> [word(W, topic) || W <- emqx_topic:tokens(Topic)]. diff --git a/apps/emqx/test/emqx_topic_index_SUITE.erl b/apps/emqx/test/emqx_topic_index_SUITE.erl index 08056a16f..9df9743f1 100644 --- a/apps/emqx/test/emqx_topic_index_SUITE.erl +++ b/apps/emqx/test/emqx_topic_index_SUITE.erl @@ -57,6 +57,17 @@ t_insert(Config) -> ?assertEqual(<<"sensor/#">>, topic(match(M, <<"sensor">>, Tab))), ?assertEqual(t_insert_3, id(match(M, <<"sensor">>, Tab))). +t_insert_filter(Config) -> + M = get_module(Config), + Tab = M:new(), + Topic = <<"sensor/+/metric//#">>, + true = M:insert(Topic, 1, <<>>, Tab), + true = M:insert(emqx_trie_search:filter(Topic), 2, <<>>, Tab), + ?assertEqual( + [Topic, Topic], + [topic(X) || X <- matches(M, <<"sensor/1/metric//2">>, Tab)] + ). + t_match(Config) -> M = get_module(Config), Tab = M:new(), diff --git a/apps/emqx/test/emqx_trie_search_tests.erl b/apps/emqx/test/emqx_trie_search_tests.erl index 75994131d..d78347de6 100644 --- a/apps/emqx/test/emqx_trie_search_tests.erl +++ b/apps/emqx/test/emqx_trie_search_tests.erl @@ -18,15 +18,30 @@ -include_lib("eunit/include/eunit.hrl"). -topic_validation_test() -> +-import(emqx_trie_search, [filter/1]). + +filter_test_() -> + [ + ?_assertEqual( + [<<"sensor">>, '+', <<"metric">>, <<>>, '#'], + filter(<<"sensor/+/metric//#">>) + ), + ?_assertEqual( + false, + filter(<<"sensor/1/metric//42">>) + ) + ]. + +topic_validation_test_() -> NextF = fun(_) -> '$end_of_table' end, Call = fun(Topic) -> emqx_trie_search:match(Topic, NextF) end, - ?assertError(badarg, Call(<<"+">>)), - ?assertError(badarg, Call(<<"#">>)), - ?assertError(badarg, Call(<<"a/+/b">>)), - ?assertError(badarg, Call(<<"a/b/#">>)), - ?assertEqual(false, Call(<<"a/b/b+">>)), - ?assertEqual(false, Call(<<"a/b/c#">>)), - ok. + [ + ?_assertError(badarg, Call(<<"+">>)), + ?_assertError(badarg, Call(<<"#">>)), + ?_assertError(badarg, Call(<<"a/+/b">>)), + ?_assertError(badarg, Call(<<"a/b/#">>)), + ?_assertEqual(false, Call(<<"a/b/b+">>)), + ?_assertEqual(false, Call(<<"a/b/c#">>)) + ]. From 063d6200c8db7fe3394f050472c6a1a60dc6bf7e Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 29 Aug 2023 17:41:59 +0400 Subject: [PATCH 088/119] feat(router): enable using 2 tables to store routes Instead of a single unified table, to reap the benefits of cheap `ets:lookup/2` per regular topic subscription route. Change configuration option naming to reflect the change: user now has an ability to choose _storage schema_. --- apps/emqx/include/emqx_router.hrl | 2 +- apps/emqx/src/emqx_app.erl | 2 +- apps/emqx/src/emqx_router.erl | 427 ++++++++++++-------- apps/emqx/src/emqx_router_sup.erl | 2 +- apps/emqx/src/emqx_schema.erl | 21 +- apps/emqx/test/emqx_router_SUITE.erl | 26 +- apps/emqx/test/emqx_router_helper_SUITE.erl | 16 +- rel/i18n/emqx_schema.hocon | 11 +- 8 files changed, 298 insertions(+), 209 deletions(-) diff --git a/apps/emqx/include/emqx_router.hrl b/apps/emqx/include/emqx_router.hrl index 99ca3e185..35a267aa7 100644 --- a/apps/emqx/include/emqx_router.hrl +++ b/apps/emqx/include/emqx_router.hrl @@ -19,7 +19,7 @@ %% ETS tables for message routing -define(ROUTE_TAB, emqx_route). --define(ROUTE_TAB_UNIFIED, emqx_route_unified). +-define(ROUTE_TAB_FILTERS, emqx_route_filters). %% Mnesia table for message routing -define(ROUTING_NODE, emqx_routing_node). diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index 4fae556ca..0f4987085 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -56,7 +56,7 @@ prep_stop(_State) -> emqx_listeners:stop(). stop(_State) -> - ok = emqx_router:deinit_table_type(), + ok = emqx_router:deinit_schema(), ok. -define(CONFIG_LOADER, config_loader). diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 7a8d88a55..9f6913dc4 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -73,9 +73,9 @@ ]). -export([ - get_table_type/0, - init_table_type/0, - deinit_table_type/0 + get_schema_vsn/0, + init_schema/0, + deinit_schema/0 ]). -type group() :: binary(). @@ -87,8 +87,6 @@ unused = [] :: nil() }). --dialyzer({nowarn_function, [cleanup_routes_regular/1]}). - %%-------------------------------------------------------------------- %% Mnesia bootstrap %%-------------------------------------------------------------------- @@ -108,7 +106,7 @@ mnesia(boot) -> ]} ]} ]), - ok = mria:create_table(?ROUTE_TAB_UNIFIED, [ + ok = mria:create_table(?ROUTE_TAB_FILTERS, [ {type, ordered_set}, {rlog_shard, ?ROUTE_SHARD}, {storage, ram_copies}, @@ -158,87 +156,41 @@ do_add_route(Topic, Dest) when is_binary(Topic) -> ok; false -> ok = emqx_router_helper:monitor(Dest), - mria_insert_route(get_table_type(), Topic, Dest) + mria_insert_route(get_schema_vsn(), Topic, Dest) end. -mria_insert_route(unified, Topic, Dest) -> - mria_insert_route_unified(Topic, Dest); -mria_insert_route(regular, Topic, Dest) -> - Route = #route{topic = Topic, dest = Dest}, - case emqx_topic:wildcard(Topic) of - true -> - mria_insert_route_update_trie(Route); - false -> - mria_insert_route(Route) - end. - -mria_insert_route_unified(Topic, Dest) -> - K = emqx_topic_index:make_key(Topic, Dest), - mria:dirty_write(?ROUTE_TAB_UNIFIED, #routeidx{entry = K}). - -mria_insert_route_update_trie(Route) -> - emqx_router_utils:maybe_trans( - fun emqx_router_utils:insert_trie_route/2, - [?ROUTE_TAB, Route], - ?ROUTE_SHARD - ). - -mria_insert_route(Route) -> - mria:dirty_write(?ROUTE_TAB, Route). +mria_insert_route(v2, Topic, Dest) -> + mria_insert_route_v2(Topic, Dest); +mria_insert_route(v1, Topic, Dest) -> + mria_insert_route_v1(Topic, Dest). %% @doc Match routes -spec match_routes(emqx_types:topic()) -> [emqx_types:route()]. match_routes(Topic) when is_binary(Topic) -> - match_routes(get_table_type(), Topic). + match_routes(get_schema_vsn(), Topic). -match_routes(unified, Topic) -> - [match_to_route(M) || M <- match_unified(Topic)]; -match_routes(regular, Topic) -> - lookup_routes_regular(Topic) ++ - lists:flatmap(fun lookup_routes_regular/1, match_global_trie(Topic)). - -match_unified(Topic) -> - emqx_topic_index:matches(Topic, ?ROUTE_TAB_UNIFIED, []). - -match_global_trie(Topic) -> - case emqx_trie:empty() of - true -> []; - false -> emqx_trie:match(Topic) - end. +match_routes(v2, Topic) -> + match_routes_v2(Topic); +match_routes(v1, Topic) -> + match_routes_v1(Topic). -spec lookup_routes(emqx_types:topic()) -> [emqx_types:route()]. lookup_routes(Topic) -> - case get_table_type() of - unified -> - lookup_routes_unified(Topic); - regular -> - lookup_routes_regular(Topic) - end. + lookup_routes(get_schema_vsn(), Topic). -lookup_routes_unified(Topic) -> - Pat = #routeidx{entry = emqx_topic_index:make_key(Topic, '$1')}, - [Dest || [Dest] <- ets:match(?ROUTE_TAB_UNIFIED, Pat)]. - -lookup_routes_regular(Topic) -> - ets:lookup(?ROUTE_TAB, Topic). - -match_to_route(M) -> - #route{topic = emqx_topic_index:get_topic(M), dest = emqx_topic_index:get_id(M)}. +lookup_routes(v2, Topic) -> + lookup_routes_v2(Topic); +lookup_routes(v1, Topic) -> + lookup_routes_v1(Topic). -spec has_route(emqx_types:topic(), dest()) -> boolean(). has_route(Topic, Dest) -> - case get_table_type() of - unified -> - has_route_unified(Topic, Dest); - regular -> - has_route_regular(Topic, Dest) - end. + has_route(get_schema_vsn(), Topic, Dest). -has_route_unified(Topic, Dest) -> - ets:member(?ROUTE_TAB_UNIFIED, emqx_topic_index:make_key(Topic, Dest)). - -has_route_regular(Topic, Dest) -> - lists:any(fun(Route) -> Route#route.dest =:= Dest end, ets:lookup(?ROUTE_TAB, Topic)). +has_route(v2, Topic, Dest) -> + has_route_v2(Topic, Dest); +has_route(v1, Topic, Dest) -> + has_route_v1(Topic, Dest). -spec delete_route(emqx_types:topic()) -> ok | {error, term()}. delete_route(Topic) when is_binary(Topic) -> @@ -254,42 +206,21 @@ do_delete_route(Topic) when is_binary(Topic) -> -spec do_delete_route(emqx_types:topic(), dest()) -> ok | {error, term()}. do_delete_route(Topic, Dest) -> - mria_delete_route(get_table_type(), Topic, Dest). + mria_delete_route(get_schema_vsn(), Topic, Dest). -mria_delete_route(unified, Topic, Dest) -> - mria_delete_route_unified(Topic, Dest); -mria_delete_route(regular, Topic, Dest) -> - Route = #route{topic = Topic, dest = Dest}, - case emqx_topic:wildcard(Topic) of - true -> - mria_delete_route_update_trie(Route); - false -> - mria_delete_route(Route) - end. - -mria_delete_route_unified(Topic, Dest) -> - K = emqx_topic_index:make_key(Topic, Dest), - mria:dirty_delete(?ROUTE_TAB_UNIFIED, K). - -mria_delete_route_update_trie(Route) -> - emqx_router_utils:maybe_trans( - fun emqx_router_utils:delete_trie_route/2, - [?ROUTE_TAB, Route], - ?ROUTE_SHARD - ). - -mria_delete_route(Route) -> - mria:dirty_delete_object(?ROUTE_TAB, Route). +mria_delete_route(v2, Topic, Dest) -> + mria_delete_route_v2(Topic, Dest); +mria_delete_route(v1, Topic, Dest) -> + mria_delete_route_v1(Topic, Dest). -spec topics() -> list(emqx_types:topic()). topics() -> - topics(get_table_type()). + topics(get_schema_vsn()). -topics(unified) -> - Pat = #routeidx{entry = '$1'}, - [emqx_topic_index:get_topic(K) || [K] <- ets:match(?ROUTE_TAB_UNIFIED, Pat)]; -topics(regular) -> - mnesia:dirty_all_keys(?ROUTE_TAB). +topics(v2) -> + list_topics_v2(); +topics(v1) -> + list_topics_v1(). %% @doc Print routes to a topic -spec print_routes(emqx_types:topic()) -> ok. @@ -303,31 +234,99 @@ print_routes(Topic) -> -spec cleanup_routes(node()) -> ok. cleanup_routes(Node) -> - case get_table_type() of - unified -> - cleanup_routes_unified(Node); - regular -> - cleanup_routes_regular(Node) + cleanup_routes(get_schema_vsn(), Node). + +cleanup_routes(v2, Node) -> + cleanup_routes_v2(Node); +cleanup_routes(v1, Node) -> + cleanup_routes_v1(Node). + +-spec foldl_routes(fun((emqx_types:route(), Acc) -> Acc), Acc) -> Acc. +foldl_routes(FoldFun, AccIn) -> + fold_routes(get_schema_vsn(), foldl, FoldFun, AccIn). + +-spec foldr_routes(fun((emqx_types:route(), Acc) -> Acc), Acc) -> Acc. +foldr_routes(FoldFun, AccIn) -> + fold_routes(get_schema_vsn(), foldr, FoldFun, AccIn). + +fold_routes(v2, FunName, FoldFun, AccIn) -> + fold_routes_v2(FunName, FoldFun, AccIn); +fold_routes(v1, FunName, FoldFun, AccIn) -> + fold_routes_v1(FunName, FoldFun, AccIn). + +call(Router, Msg) -> + gen_server:call(Router, Msg, infinity). + +pick(Topic) -> + gproc_pool:pick_worker(router_pool, Topic). + +%%-------------------------------------------------------------------- +%% Schema v1 +%% -------------------------------------------------------------------- + +-dialyzer({nowarn_function, [cleanup_routes_v1/1]}). + +mria_insert_route_v1(Topic, Dest) -> + Route = #route{topic = Topic, dest = Dest}, + case emqx_topic:wildcard(Topic) of + true -> + mria_route_tab_insert_update_trie(Route); + false -> + mria_route_tab_insert(Route) end. -cleanup_routes_unified(Node) -> - % NOTE - % No point in transaction here because all the operations on unified routing table - % are dirty. - ets:foldl( - fun(#routeidx{entry = K}, ok) -> - case emqx_topic_index:get_id(K) of - Node -> - mria:dirty_delete(?ROUTE_TAB_UNIFIED, K); - _ -> - ok - end - end, - ok, - ?ROUTE_TAB_UNIFIED +mria_route_tab_insert_update_trie(Route) -> + emqx_router_utils:maybe_trans( + fun emqx_router_utils:insert_trie_route/2, + [?ROUTE_TAB, Route], + ?ROUTE_SHARD ). -cleanup_routes_regular(Node) -> +mria_route_tab_insert(Route) -> + mria:dirty_write(?ROUTE_TAB, Route). + +mria_delete_route_v1(Topic, Dest) -> + Route = #route{topic = Topic, dest = Dest}, + case emqx_topic:wildcard(Topic) of + true -> + mria_route_tab_delete_update_trie(Route); + false -> + mria_route_tab_delete(Route) + end. + +mria_route_tab_delete_update_trie(Route) -> + emqx_router_utils:maybe_trans( + fun emqx_router_utils:delete_trie_route/2, + [?ROUTE_TAB, Route], + ?ROUTE_SHARD + ). + +mria_route_tab_delete(Route) -> + mria:dirty_delete_object(?ROUTE_TAB, Route). + +match_routes_v1(Topic) -> + lookup_route_tab(Topic) ++ + lists:flatmap(fun lookup_route_tab/1, match_global_trie(Topic)). + +match_global_trie(Topic) -> + case emqx_trie:empty() of + true -> []; + false -> emqx_trie:match(Topic) + end. + +lookup_routes_v1(Topic) -> + lookup_route_tab(Topic). + +lookup_route_tab(Topic) -> + ets:lookup(?ROUTE_TAB, Topic). + +has_route_v1(Topic, Dest) -> + has_route_tab_entry(Topic, Dest). + +has_route_tab_entry(Topic, Dest) -> + [] =/= ets:match(?ROUTE_TAB, #route{topic = Topic, dest = Dest}). + +cleanup_routes_v1(Node) -> Patterns = [ #route{_ = '_', dest = Node}, #route{_ = '_', dest = {'_', Node}} @@ -340,83 +339,165 @@ cleanup_routes_regular(Node) -> ] end). --spec foldl_routes(fun((emqx_types:route(), Acc) -> Acc), Acc) -> Acc. -foldl_routes(FoldFun, AccIn) -> - case get_table_type() of - unified -> - ets:foldl(mk_fold_fun_unified(FoldFun), AccIn, ?ROUTE_TAB_UNIFIED); - regular -> - ets:foldl(FoldFun, AccIn, ?ROUTE_TAB) +list_topics_v1() -> + list_route_tab_topics(). + +list_route_tab_topics() -> + mnesia:dirty_all_keys(?ROUTE_TAB). + +fold_routes_v1(FunName, FoldFun, AccIn) -> + ets:FunName(FoldFun, AccIn, ?ROUTE_TAB). + +%%-------------------------------------------------------------------- +%% Schema v2 +%% One bag table exclusively for regular, non-filter subscription +%% topics, and one `emqx_topic_index` table exclusively for wildcard +%% topics. Writes go to only one of the two tables at a time. +%% -------------------------------------------------------------------- + +mria_insert_route_v2(Topic, Dest) -> + case emqx_trie_search:filter(Topic) of + Words when is_list(Words) -> + K = emqx_topic_index:make_key(Words, Dest), + mria:dirty_write(?ROUTE_TAB_FILTERS, #routeidx{entry = K}); + false -> + mria_route_tab_insert(#route{topic = Topic, dest = Dest}) end. --spec foldr_routes(fun((emqx_types:route(), Acc) -> Acc), Acc) -> Acc. -foldr_routes(FoldFun, AccIn) -> - case get_table_type() of - unified -> - ets:foldr(mk_fold_fun_unified(FoldFun), AccIn, ?ROUTE_TAB_UNIFIED); - regular -> - ets:foldr(FoldFun, AccIn, ?ROUTE_TAB) +mria_delete_route_v2(Topic, Dest) -> + case emqx_trie_search:filter(Topic) of + Words when is_list(Words) -> + K = emqx_topic_index:make_key(Words, Dest), + mria:dirty_delete(?ROUTE_TAB_FILTERS, K); + false -> + mria_route_tab_delete(#route{topic = Topic, dest = Dest}) end. -mk_fold_fun_unified(FoldFun) -> +match_routes_v2(Topic) -> + lookup_route_tab(Topic) ++ + [match_to_route(M) || M <- match_filters(Topic)]. + +match_filters(Topic) -> + emqx_topic_index:matches(Topic, ?ROUTE_TAB_FILTERS, []). + +lookup_routes_v2(Topic) -> + case emqx_topic:wildcard(Topic) of + true -> + Pat = #routeidx{entry = emqx_topic_index:make_key(Topic, '$1')}, + [Dest || [Dest] <- ets:match(?ROUTE_TAB_FILTERS, Pat)]; + false -> + lookup_route_tab(Topic) + end. + +has_route_v2(Topic, Dest) -> + case emqx_topic:wildcard(Topic) of + true -> + ets:member(?ROUTE_TAB_FILTERS, emqx_topic_index:make_key(Topic, Dest)); + false -> + has_route_tab_entry(Topic, Dest) + end. + +cleanup_routes_v2(Node) -> + % NOTE + % No point in transaction here because all the operations on unified routing table + % are dirty. + ok = ets:foldl( + fun(#routeidx{entry = K}, ok) -> + case get_dest_node(emqx_topic_index:get_id(K)) of + Node -> + mria:dirty_delete(?ROUTE_TAB_FILTERS, K); + _ -> + ok + end + end, + ok, + ?ROUTE_TAB_FILTERS + ), + ok = ets:foldl( + fun(#route{dest = Dest} = Route, ok) -> + case get_dest_node(Dest) of + Node -> + mria:dirty_delete_object(?ROUTE_TAB, Route); + _ -> + ok + end + end, + ok, + ?ROUTE_TAB + ). + +get_dest_node({_, Node}) -> + Node; +get_dest_node(Node) -> + Node. + +list_topics_v2() -> + Pat = #routeidx{entry = '$1'}, + Filters = [emqx_topic_index:get_topic(K) || [K] <- ets:match(?ROUTE_TAB_FILTERS, Pat)], + list_route_tab_topics() ++ Filters. + +fold_routes_v2(FunName, FoldFun, AccIn) -> + FilterFoldFun = mk_filtertab_fold_fun(FoldFun), + Acc = ets:FunName(FoldFun, AccIn, ?ROUTE_TAB), + ets:FunName(FilterFoldFun, Acc, ?ROUTE_TAB_FILTERS). + +mk_filtertab_fold_fun(FoldFun) -> fun(#routeidx{entry = K}, Acc) -> FoldFun(match_to_route(K), Acc) end. -call(Router, Msg) -> - gen_server:call(Router, Msg, infinity). - -pick(Topic) -> - gproc_pool:pick_worker(router_pool, Topic). +match_to_route(M) -> + #route{topic = emqx_topic_index:get_topic(M), dest = emqx_topic_index:get_id(M)}. %%-------------------------------------------------------------------- %% Routing table type %% -------------------------------------------------------------------- --define(PT_TABLE_TYPE, {?MODULE, tabtype}). +-define(PT_SCHEMA_VSN, {?MODULE, schemavsn}). --type tabtype() :: regular | unified. +-type schemavsn() :: v1 | v2. --spec get_table_type() -> tabtype(). -get_table_type() -> - persistent_term:get(?PT_TABLE_TYPE). +-spec get_schema_vsn() -> schemavsn(). +get_schema_vsn() -> + persistent_term:get(?PT_SCHEMA_VSN). --spec init_table_type() -> ok. -init_table_type() -> - ConfType = emqx_config:get([broker, routing_table_type]), - Type = choose_table_type(ConfType), - ok = persistent_term:put(?PT_TABLE_TYPE, Type), - case Type of - ConfType -> +-spec init_schema() -> ok. +init_schema() -> + ConfSchema = emqx_config:get([broker, routing, storage_schema]), + Schema = choose_schema_vsn(ConfSchema), + ok = persistent_term:put(?PT_SCHEMA_VSN, Schema), + case Schema of + ConfSchema -> ?SLOG(info, #{ - msg => "routing_table_type_used", - type => Type + msg => "routing_schema_used", + schema => Schema }); _ -> ?SLOG(notice, #{ - msg => "configured_routing_table_type_unacceptable", - type => Type, - configured => ConfType, + msg => "configured_routing_schema_unacceptable", + schema => Schema, + configured => ConfSchema, reason => - "Could not use configured routing table type because " - "there's already non-empty routing table of another type." + "Could not use configured routing storage schema because " + "there are already non-empty routing tables pertaining to " + "another schema." }) end. --spec deinit_table_type() -> ok. -deinit_table_type() -> - _ = persistent_term:erase(?PT_TABLE_TYPE), +-spec deinit_schema() -> ok. +deinit_schema() -> + _ = persistent_term:erase(?PT_SCHEMA_VSN), ok. --spec choose_table_type(tabtype()) -> tabtype(). -choose_table_type(ConfType) -> - IsEmptyRegular = is_empty(?ROUTE_TAB), - IsEmptyUnified = is_empty(?ROUTE_TAB_UNIFIED), - case {IsEmptyRegular, IsEmptyUnified} of +-spec choose_schema_vsn(schemavsn()) -> schemavsn(). +choose_schema_vsn(ConfType) -> + IsEmptyIndex = emqx_trie:empty(), + IsEmptyFilters = is_empty(?ROUTE_TAB_FILTERS), + case {IsEmptyIndex, IsEmptyFilters} of {true, true} -> ConfType; {false, true} -> - regular; + v1; {true, false} -> - unified + v2 end. is_empty(Tab) -> diff --git a/apps/emqx/src/emqx_router_sup.erl b/apps/emqx/src/emqx_router_sup.erl index 398edc321..588b0de8e 100644 --- a/apps/emqx/src/emqx_router_sup.erl +++ b/apps/emqx/src/emqx_router_sup.erl @@ -24,7 +24,7 @@ start_link() -> %% Init and log routing table type - ok = emqx_router:init_table_type(), + ok = emqx_router:init_schema(), supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 818688d86..85bdd7b50 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1358,14 +1358,10 @@ fields("broker") -> ref("broker_perf"), #{importance => ?IMPORTANCE_HIDDEN} )}, - {"routing_table_type", + {"routing", sc( - hoconsc:enum([regular, unified]), - #{ - default => regular, - importance => ?IMPORTANCE_HIDDEN, - desc => ?DESC(broker_routing_table_type) - } + ref("broker_routing"), + #{importance => ?IMPORTANCE_HIDDEN} )}, %% FIXME: Need new design for shared subscription group {"shared_subscription_group", @@ -1378,6 +1374,17 @@ fields("broker") -> } )} ]; +fields("broker_routing") -> + [ + {"storage_schema", + sc( + hoconsc:enum([v1, v2]), + #{ + default => v1, + desc => ?DESC(broker_routing_storage_schema) + } + )} + ]; fields("shared_subscription_group") -> [ {"strategy", diff --git a/apps/emqx/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl index f9e1c6998..adc898365 100644 --- a/apps/emqx/test/emqx_router_SUITE.erl +++ b/apps/emqx/test/emqx_router_SUITE.erl @@ -28,15 +28,15 @@ all() -> [ - {group, routing_table_regular}, - {group, routing_table_unified} + {group, routing_schema_v1}, + {group, routing_schema_v2} ]. groups() -> TCs = emqx_common_test_helpers:all(?MODULE), [ - {routing_table_regular, [], TCs}, - {routing_table_unified, [], TCs} + {routing_schema_v1, [], TCs}, + {routing_schema_v2, [], TCs} ]. init_per_group(GroupName, Config) -> @@ -53,10 +53,10 @@ init_per_group(GroupName, Config) -> end_per_group(_GroupName, Config) -> ok = emqx_cth_suite:stop(?config(group_apps, Config)). -mk_config(routing_table_regular) -> - "broker.routing_table_type = regular"; -mk_config(routing_table_unified) -> - "broker.routing_table_type = unified". +mk_config(routing_schema_v1) -> + "broker.routing.storage_schema = v1"; +mk_config(routing_schema_v2) -> + "broker.routing.storage_schema = v2". init_per_testcase(_TestCase, Config) -> clear_tables(), @@ -85,10 +85,10 @@ end_per_testcase(_TestCase, _Config) -> t_verify_type(Config) -> case ?config(group_name, Config) of - routing_table_regular -> - ?assertEqual(regular, ?R:get_table_type()); - routing_table_unified -> - ?assertEqual(unified, ?R:get_table_type()) + routing_schema_v1 -> + ?assertEqual(v1, ?R:get_schema_vsn()); + routing_schema_v2 -> + ?assertEqual(v2, ?R:get_schema_vsn()) end. t_add_delete(_) -> @@ -198,5 +198,5 @@ t_unexpected(_) -> clear_tables() -> lists:foreach( fun mnesia:clear_table/1, - [?ROUTE_TAB, ?ROUTE_TAB_UNIFIED, ?TRIE] + [?ROUTE_TAB, ?ROUTE_TAB_FILTERS, ?TRIE] ). diff --git a/apps/emqx/test/emqx_router_helper_SUITE.erl b/apps/emqx/test/emqx_router_helper_SUITE.erl index 26b9d8ddd..f4d3cc43a 100644 --- a/apps/emqx/test/emqx_router_helper_SUITE.erl +++ b/apps/emqx/test/emqx_router_helper_SUITE.erl @@ -28,15 +28,15 @@ all() -> [ - {group, routing_table_regular}, - {group, routing_table_unified} + {group, routing_schema_v1}, + {group, routing_schema_v2} ]. groups() -> TCs = emqx_common_test_helpers:all(?MODULE), [ - {routing_table_regular, [], TCs}, - {routing_table_unified, [], TCs} + {routing_schema_v1, [], TCs}, + {routing_schema_v2, [], TCs} ]. init_per_group(GroupName, Config) -> @@ -48,14 +48,14 @@ init_per_group(GroupName, Config) -> end_per_group(_GroupName, Config) -> ok = emqx_cth_suite:stop(?config(group_apps, Config)). -mk_config(routing_table_regular) -> +mk_config(routing_schema_v1) -> #{ - config => "broker.routing_table_type = regular", + config => "broker.routing.storage_schema = v1", override_env => [{boot_modules, [router]}] }; -mk_config(routing_table_unified) -> +mk_config(routing_schema_v2) -> #{ - config => "broker.routing_table_type = unified", + config => "broker.routing.storage_schema = v2", override_env => [{boot_modules, [router]}] }. diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index d90fab47b..4a8440e83 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1549,11 +1549,12 @@ fields_ws_opts_max_frame_size.label: sys_event_messages.desc: """Client events messages.""" -broker_routing_table_type.desc: -"""Routing table type. -Unified routing table should increase both subscription and routing performance at the cost of slight increase in memory consumption per subscription. -NOTE: This is an experimental feature. -NOTE: Full non-rolling cluster restart is needed after enabling or disabling this option for it to take any effect.""" +broker_routing_storage_schema.desc: +"""Routing storage schema. +Set v1 to leave the default. +Set v2 to enable routing through 2 separate tables, one for topic filter and one for regular topic subscriptions. This schema should increase both subscription and routing performance at the cost of slight increase in memory consumption per subscription. +NOTE: Schema v2 is still experimental. +NOTE: Full non-rolling cluster restart is needed after altering this option for it to take any effect.""" broker_perf_trie_compaction.desc: """Enable trie path compaction. From 0f297ffef477d6e1f37122266f2baa4e91579dc5 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 29 Aug 2023 14:43:24 -0300 Subject: [PATCH 089/119] fix(bridge): validate bridge name before attempting to convert certificates Fixes https://emqx.atlassian.net/browse/EMQX-10865 --- apps/emqx_bridge/src/emqx_bridge.erl | 20 ++++++++ apps/emqx_bridge/src/emqx_bridge_api.erl | 2 + apps/emqx_bridge/src/emqx_bridge_app.erl | 24 +++++++-- apps/emqx_bridge/test/emqx_bridge_SUITE.erl | 50 ++++++++++++++----- .../test/emqx_bridge_api_SUITE.erl | 37 ++++++++++++++ changes/ce/fix-11540.en.md | 1 + 6 files changed, 117 insertions(+), 17 deletions(-) create mode 100644 changes/ce/fix-11540.en.md diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index edd21a239..4fa6cd346 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -55,6 +55,7 @@ ]). -export([config_key_path/0]). +-export([validate_bridge_name/1]). %% exported for `emqx_telemetry' -export([get_basic_usage_info/0]). @@ -96,6 +97,9 @@ -define(ROOT_KEY, bridges). +%% See `hocon_tconf` +-define(MAP_KEY_RE, <<"^[A-Za-z0-9]+[A-Za-z0-9-_]*$">>). + load() -> Bridges = emqx:get_config([?ROOT_KEY], #{}), lists:foreach( @@ -580,3 +584,19 @@ get_basic_usage_info() -> _:_ -> InitialAcc end. + +validate_bridge_name(BridgeName0) -> + BridgeName = to_bin(BridgeName0), + case re:run(BridgeName, ?MAP_KEY_RE, [{capture, none}]) of + match -> + ok; + nomatch -> + {error, #{ + kind => validation_error, + reason => bad_bridge_name, + value => BridgeName + }} + end. + +to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +to_bin(B) when is_binary(B) -> B. diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 3190a2ef9..e49b54d67 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -609,6 +609,8 @@ create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) -> case emqx_bridge:create(BridgeType, BridgeName, Conf) of {ok, _} -> lookup_from_all_nodes(BridgeType, BridgeName, HttpStatusCode); + {error, {pre_config_update, _HandlerMod, Reason}} when is_map(Reason) -> + ?BAD_REQUEST(map_to_json(redact(Reason))); {error, Reason} when is_map(Reason) -> ?BAD_REQUEST(map_to_json(redact(Reason))) end. diff --git a/apps/emqx_bridge/src/emqx_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl index 3bae55090..d0dd7da2b 100644 --- a/apps/emqx_bridge/src/emqx_bridge_app.erl +++ b/apps/emqx_bridge/src/emqx_bridge_app.erl @@ -62,11 +62,16 @@ pre_config_update(_, {Oper, _Type, _Name}, OldConfig) -> %% to save the 'enable' to the config files {ok, OldConfig#{<<"enable">> => operation_to_enable(Oper)}}; pre_config_update(Path, Conf, _OldConfig) when is_map(Conf) -> - case emqx_connector_ssl:convert_certs(filename:join(Path), Conf) of - {error, Reason} -> - {error, Reason}; - {ok, ConfNew} -> - {ok, ConfNew} + case validate_bridge_name(Path) of + ok -> + case emqx_connector_ssl:convert_certs(filename:join(Path), Conf) of + {error, Reason} -> + {error, Reason}; + {ok, ConfNew} -> + {ok, ConfNew} + end; + Error -> + Error end. post_config_update([bridges, BridgeType, BridgeName], '$remove', _, _OldConf, _AppEnvs) -> @@ -97,3 +102,12 @@ post_config_update([bridges, BridgeType, BridgeName], _Req, NewConf, OldConf, _A %% internal functions operation_to_enable(disable) -> false; operation_to_enable(enable) -> true. + +validate_bridge_name(Path) -> + [RootKey] = emqx_bridge:config_key_path(), + case Path of + [RootKey, _BridgeType, BridgeName] -> + emqx_bridge:validate_bridge_name(BridgeName); + _ -> + ok + end. diff --git a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl index 1cbc94d24..c9157d9e6 100644 --- a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl @@ -26,19 +26,20 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - _ = application:load(emqx_conf), - %% to avoid inter-suite dependencies - application:stop(emqx_connector), - ok = emqx_common_test_helpers:start_apps([emqx, emqx_bridge]), - Config. + Apps = emqx_cth_suite:start( + [ + emqx, + emqx_conf, + emqx_bridge + ], + #{work_dir => ?config(priv_dir, Config)} + ), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([ - emqx, - emqx_bridge, - emqx_resource, - emqx_connector - ]). +end_per_suite(Config) -> + Apps = ?config(apps, Config), + ok = emqx_cth_suite:stop(Apps), + ok. init_per_testcase(t_get_basic_usage_info_1, Config) -> {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), @@ -180,6 +181,31 @@ t_update_ssl_conf(Config) -> ?assertMatch({error, enoent}, file:list_dir(CertDir)), ok. +t_create_with_bad_name(_Config) -> + Path = [bridges, mqtt, 'test_哈哈'], + Conf = #{ + <<"bridge_mode">> => false, + <<"clean_start">> => true, + <<"keepalive">> => <<"60s">>, + <<"proto_ver">> => <<"v4">>, + <<"server">> => <<"127.0.0.1:1883">>, + <<"ssl">> => + #{ + %% needed to trigger pre_config_update + <<"certfile">> => cert_file("certfile"), + <<"enable">> => true + } + }, + ?assertMatch( + {error, + {pre_config_update, emqx_bridge_app, #{ + reason := bad_bridge_name, + kind := validation_error + }}}, + emqx:update_config(Path, Conf) + ), + ok. + data_file(Name) -> Dir = code:lib_dir(emqx_bridge, test), {ok, Bin} = file:read_file(filename:join([Dir, "data", Name])), diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index d8e697987..31c6bcab1 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -1335,6 +1335,35 @@ t_cluster_later_join_metrics(Config) -> ), ok. +t_create_with_bad_name(Config) -> + Port = ?config(port, Config), + URL1 = ?URL(Port, "path1"), + Name = <<"test_哈哈">>, + BadBridgeParams = + emqx_utils_maps:deep_merge( + ?HTTP_BRIDGE(URL1, Name), + #{ + <<"ssl">> => + #{ + <<"enable">> => true, + <<"certfile">> => cert_file("certfile") + } + } + ), + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := Msg0 + }} = + request_json( + post, + uri(["bridges"]), + BadBridgeParams, + Config + ), + Msg = emqx_utils_json:decode(Msg0, [return_maps]), + ?assertMatch(#{<<"reason">> := <<"bad_bridge_name">>}, Msg), + ok. + validate_resource_request_ttl(single, Timeout, Name) -> SentData = #{payload => <<"Hello EMQX">>, timestamp => 1668602148000}, BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), @@ -1418,3 +1447,11 @@ str(S) when is_binary(S) -> binary_to_list(S). json(B) when is_binary(B) -> emqx_utils_json:decode(B, [return_maps]). + +data_file(Name) -> + Dir = code:lib_dir(emqx_bridge, test), + {ok, Bin} = file:read_file(filename:join([Dir, "data", Name])), + Bin. + +cert_file(Name) -> + data_file(filename:join(["certs", Name])). diff --git a/changes/ce/fix-11540.en.md b/changes/ce/fix-11540.en.md new file mode 100644 index 000000000..7486e7133 --- /dev/null +++ b/changes/ce/fix-11540.en.md @@ -0,0 +1 @@ +Improved HTTP response when attempting to create a bridge with an invalid name. From dfcede8794a2804a4648e256653e45f47688a73b Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Wed, 30 Aug 2023 10:52:53 -0300 Subject: [PATCH 090/119] fix: increment matched counter when bridge is unhealthy Fixes https://emqx.atlassian.net/browse/EMQX-10767 --- apps/emqx_resource/src/emqx_resource.erl | 2 ++ apps/emqx_resource/test/emqx_resource_SUITE.erl | 13 +++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 1e0f0b45d..d41364315 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -281,9 +281,11 @@ query(ResId, Request, Opts) -> {ok, _Group, #{query_mode := QM, error := Error}} -> case {QM, Error} of {_, unhealthy_target} -> + emqx_resource_metrics:matched_inc(ResId), emqx_resource_metrics:dropped_resource_stopped_inc(ResId), ?RESOURCE_ERROR(unhealthy_target, "unhealthy target"); {_, {unhealthy_target, _Message}} -> + emqx_resource_metrics:matched_inc(ResId), emqx_resource_metrics:dropped_resource_stopped_inc(ResId), ?RESOURCE_ERROR(unhealthy_target, "unhealthy target"); {simple_async, _} -> diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index d4bc9a8a7..ef9ee29c7 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -919,8 +919,17 @@ t_unhealthy_target(_) -> emqx_resource_manager:lookup(?ID) ), %% messages are dropped when bridge is unhealthy - emqx_resource:query(?ID, message), - ?assertEqual(1, emqx_resource_metrics:dropped_resource_stopped_get(?ID)). + lists:foreach( + fun(_) -> + ?assertMatch( + {error, {resource_error, #{reason := unhealthy_target}}}, + emqx_resource:query(?ID, message) + ) + end, + lists:seq(1, 3) + ), + ?assertEqual(3, emqx_resource_metrics:matched_get(?ID)), + ?assertEqual(3, emqx_resource_metrics:dropped_resource_stopped_get(?ID)). t_stop_start(_) -> ?check_trace( From 6b9cb063348f4225627731c47ac82431c3aa5594 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 30 Aug 2023 19:51:00 +0400 Subject: [PATCH 091/119] fix(router): add / refine some comments and log messages Co-authored-by: Zaiming (Stone) Shi --- apps/emqx/src/emqx_router.erl | 12 +++++++----- apps/emqx/src/emqx_router_helper.erl | 1 - 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 9f6913dc4..8f2947ba2 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -164,7 +164,8 @@ mria_insert_route(v2, Topic, Dest) -> mria_insert_route(v1, Topic, Dest) -> mria_insert_route_v1(Topic, Dest). -%% @doc Match routes +%% @doc Take a real topic (not filter) as input, return the matching topics and topic +%% filters associated with route destination. -spec match_routes(emqx_types:topic()) -> [emqx_types:route()]. match_routes(Topic) when is_binary(Topic) -> match_routes(get_schema_vsn(), Topic). @@ -174,6 +175,8 @@ match_routes(v2, Topic) -> match_routes(v1, Topic) -> match_routes_v1(Topic). +%% @doc Take a topic or filter as input, and return the existing routes with exactly +%% this topic or filter. -spec lookup_routes(emqx_types:topic()) -> [emqx_types:route()]. lookup_routes(Topic) -> lookup_routes(get_schema_vsn(), Topic). @@ -399,8 +402,7 @@ has_route_v2(Topic, Dest) -> cleanup_routes_v2(Node) -> % NOTE - % No point in transaction here because all the operations on unified routing table - % are dirty. + % No point in transaction here because all the operations on filters table are dirty. ok = ets:foldl( fun(#routeidx{entry = K}, ok) -> case get_dest_node(emqx_topic_index:get_id(K)) of @@ -472,8 +474,8 @@ init_schema() -> }); _ -> ?SLOG(notice, #{ - msg => "configured_routing_schema_unacceptable", - schema => Schema, + msg => "configured_routing_schema_ignored", + schema_in_use => Schema, configured => ConfSchema, reason => "Could not use configured routing storage schema because " diff --git a/apps/emqx/src/emqx_router_helper.erl b/apps/emqx/src/emqx_router_helper.erl index 77f1cd11b..b9cdbae4b 100644 --- a/apps/emqx/src/emqx_router_helper.erl +++ b/apps/emqx/src/emqx_router_helper.erl @@ -151,7 +151,6 @@ handle_info({nodedown, Node}, State = #{nodes := Nodes}) -> % TODO % Node may flap, do we need to wait for any pending cleanups in `init/1` % on the flapping node? - % This also implies changing lock id to `{?LOCK, Node}`. global:trans( {?LOCK, self()}, fun() -> cleanup_routes(Node) end From b1defa29d7597c3e81701c85961b74d5bff904b3 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 30 Aug 2023 20:07:56 +0400 Subject: [PATCH 092/119] feat(ruleeng): avoid storing whole rules in topic index Because it doesn't really give any benefit, but wastes memory by duplication. --- apps/emqx_rule_engine/src/emqx_rule_engine.erl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 0060ed819..01b9b76d0 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -227,7 +227,7 @@ get_rules_for_topic(Topic) -> [ Rule || M <- emqx_topic_index:matches(Topic, ?RULE_TOPIC_INDEX, [unique]), - Rule <- emqx_topic_index:get_record(M, ?RULE_TOPIC_INDEX) + Rule <- lookup_rule(emqx_topic_index:get_id(M)) ]. -spec get_rules_with_same_event(Topic :: binary()) -> [rule()]. @@ -285,11 +285,14 @@ is_of_event_name(EventName, Topic) -> -spec get_rule(Id :: rule_id()) -> {ok, rule()} | not_found. get_rule(Id) -> - case ets:lookup(?RULE_TAB, Id) of - [{Id, Rule}] -> {ok, Rule#{id => Id}}; + case lookup_rule(Id) of + [Rule] -> {ok, Rule}; [] -> not_found end. +lookup_rule(Id) -> + [Rule || {_Id, Rule} <- ets:lookup(?RULE_TAB, Id)]. + load_hooks_for_rule(#{from := Topics}) -> lists:foreach(fun emqx_rule_events:load/1, Topics). @@ -484,7 +487,7 @@ with_parsed_rule(Params = #{id := RuleId, sql := Sql, actions := Actions}, Creat do_insert_rule(#{id := Id} = Rule) -> ok = load_hooks_for_rule(Rule), ok = maybe_add_metrics_for_rule(Id), - true = ets:insert(?RULE_TAB, {Id, maps:remove(id, Rule)}), + true = ets:insert(?RULE_TAB, {Id, Rule}), ok. do_delete_rule(#{id := Id} = Rule) -> @@ -493,10 +496,10 @@ do_delete_rule(#{id := Id} = Rule) -> true = ets:delete(?RULE_TAB, Id), ok. -do_update_rule_index(#{id := Id, from := From} = Rule) -> +do_update_rule_index(#{id := Id, from := From}) -> ok = lists:foreach( fun(Topic) -> - true = emqx_topic_index:insert(Topic, Id, Rule, ?RULE_TOPIC_INDEX) + true = emqx_topic_index:insert(Topic, Id, [], ?RULE_TOPIC_INDEX) end, From ). From ee77976424c3dc760e91ac40e07295ac3e82bace Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Tue, 29 Aug 2023 19:26:58 -0300 Subject: [PATCH 093/119] fix(kafka): avoid producer name collision between Kafka and AEH bridges Fixes https://emqx.atlassian.net/browse/EMQX-10860 --- apps/emqx_bridge/src/emqx_bridge_resource.erl | 23 +++++-------- .../src/emqx_bridge_azure_event_hub.erl | 4 +-- ..._bridge_azure_event_hub_producer_SUITE.erl | 4 +++ .../src/emqx_bridge_kafka_impl.erl | 11 +++--- .../src/emqx_bridge_kafka_impl_consumer.erl | 12 ++++--- .../src/emqx_bridge_kafka_impl_producer.erl | 34 +++++++------------ .../emqx_bridge_kafka_impl_consumer_SUITE.erl | 7 ++-- .../emqx_bridge_kafka_impl_producer_SUITE.erl | 11 +++--- 8 files changed, 49 insertions(+), 57 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index a48d0294e..65e05b950 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -175,14 +175,15 @@ create(BridgeId, Conf) -> create(Type, Name, Conf) -> create(Type, Name, Conf, #{}). -create(Type, Name, Conf, Opts) -> +create(Type, Name, Conf0, Opts) -> ?SLOG(info, #{ msg => "create bridge", type => Type, name => Name, - config => emqx_utils:redact(Conf) + config => emqx_utils:redact(Conf0) }), TypeBin = bin(Type), + Conf = Conf0#{bridge_type => TypeBin, bridge_name => Name}, {ok, _Data} = emqx_resource:create_local( resource_id(Type, Name), <<"emqx_bridge">>, @@ -249,8 +250,9 @@ recreate(Type, Name) -> recreate(Type, Name, Conf) -> recreate(Type, Name, Conf, #{}). -recreate(Type, Name, Conf, Opts) -> +recreate(Type, Name, Conf0, Opts) -> TypeBin = bin(Type), + Conf = Conf0#{bridge_type => TypeBin, bridge_name => Name}, emqx_resource:recreate_local( resource_id(Type, Name), bridge_to_resource_type(Type), @@ -267,17 +269,18 @@ create_dry_run(Type, Conf0) -> Conf1 = maps:without([<<"name">>], Conf0), RawConf = #{<<"bridges">> => #{TypeBin => #{<<"temp_name">> => Conf1}}}, try - #{bridges := #{TypeAtom := #{temp_name := Conf}}} = + #{bridges := #{TypeAtom := #{temp_name := Conf2}}} = hocon_tconf:check_plain( emqx_bridge_schema, RawConf, #{atom_key => true, required => false} ), + Conf = Conf2#{bridge_type => TypeBin, bridge_name => TmpName}, case emqx_connector_ssl:convert_certs(TmpPath, Conf) of {error, Reason} -> {error, Reason}; {ok, ConfNew} -> - ParseConf = parse_confs(bin(Type), TmpName, ConfNew), + ParseConf = parse_confs(TypeBin, TmpName, ConfNew), emqx_resource:create_dry_run_local(bridge_to_resource_type(Type), ParseConf) end catch @@ -387,15 +390,7 @@ parse_confs(Type, Name, Conf) when ?IS_INGRESS_BRIDGE(Type) -> %% receives a message from the external database. BId = bridge_id(Type, Name), BridgeHookpoint = bridge_hookpoint(BId), - Conf#{hookpoint => BridgeHookpoint, bridge_name => Name}; -%% TODO: rename this to `kafka_producer' after alias support is added -%% to hocon; keeping this as just `kafka' for backwards compatibility. -parse_confs(<<"kafka">> = _Type, Name, Conf) -> - Conf#{bridge_name => Name}; -parse_confs(<<"pulsar_producer">> = _Type, Name, Conf) -> - Conf#{bridge_name => Name}; -parse_confs(<<"kinesis_producer">> = _Type, Name, Conf) -> - Conf#{bridge_name => Name}; + Conf#{hookpoint => BridgeHookpoint}; parse_confs(BridgeType, BridgeName, Config) -> connector_config(BridgeType, BridgeName, Config). diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl index c563a35d8..b34d6132a 100644 --- a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl @@ -166,14 +166,14 @@ values(producer) -> %% `emqx_bridge_resource' API %%------------------------------------------------------------------------------------------------- -connector_config(Config, BridgeName) -> +connector_config(Config, _BridgeName) -> %% Default port for AEH is 9093 BootstrapHosts0 = maps:get(bootstrap_hosts, Config), BootstrapHosts = emqx_schema:parse_servers( BootstrapHosts0, emqx_bridge_azure_event_hub:host_opts() ), - Config#{bridge_name => BridgeName, bootstrap_hosts := BootstrapHosts}. + Config#{bootstrap_hosts := BootstrapHosts}. %%------------------------------------------------------------------------------------------------- %% Internal fns diff --git a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_producer_SUITE.erl b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_producer_SUITE.erl index 91016295f..87c2127c2 100644 --- a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_producer_SUITE.erl +++ b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_producer_SUITE.erl @@ -282,6 +282,7 @@ t_same_name_azure_kafka_bridges(AehConfig) -> ConfigKafka = lists:keyreplace(bridge_type, 1, AehConfig, {bridge_type, ?KAFKA_BRIDGE_TYPE}), BridgeName = ?config(bridge_name, AehConfig), AehResourceId = emqx_bridge_testlib:resource_id(AehConfig), + KafkaResourceId = emqx_bridge_testlib:resource_id(ConfigKafka), TracePoint = emqx_bridge_kafka_impl_producer_sync_query, %% creates the AEH bridge and check it's working ok = emqx_bridge_testlib:t_sync_query( @@ -292,6 +293,9 @@ t_same_name_azure_kafka_bridges(AehConfig) -> ), %% than creates a Kafka bridge with same name and delete it after creation ok = emqx_bridge_testlib:t_create_via_http(ConfigKafka), + %% check that both bridges are healthy + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(AehResourceId)), + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(KafkaResourceId)), ?assertMatch( {{ok, _}, {ok, _}}, ?wait_async_action( diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl index 64ac8cf99..7532ba963 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl @@ -7,7 +7,7 @@ -export([ hosts/1, - make_client_id/1, + make_client_id/2, sasl/1, socket_opts/1 ]). @@ -24,11 +24,10 @@ hosts(Hosts) when is_list(Hosts) -> kpro:parse_endpoints(Hosts). %% Client ID is better to be unique to make it easier for Kafka side trouble shooting. -make_client_id(InstanceId) -> - InstanceIdBin0 = to_bin(InstanceId), - % Removing the <<"bridge:">> from beginning for backward compatibility - InstanceIdBin = binary:replace(InstanceIdBin0, <<"bridge:">>, <<>>), - iolist_to_binary([InstanceIdBin, ":", atom_to_list(node())]). +make_client_id(BridgeType0, BridgeName0) -> + BridgeType = to_bin(BridgeType0), + BridgeName = to_bin(BridgeName0), + iolist_to_binary([BridgeType, ":", BridgeName, ":", atom_to_list(node())]). sasl(none) -> undefined; diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl index 06176ac93..b16f163fb 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl @@ -121,6 +121,8 @@ on_start(ResourceId, Config) -> #{ authentication := Auth, bootstrap_hosts := BootstrapHosts0, + bridge_type := BridgeType, + bridge_name := BridgeName, hookpoint := _, kafka := #{ max_batch_bytes := _, @@ -134,7 +136,7 @@ on_start(ResourceId, Config) -> } = Config, BootstrapHosts = emqx_bridge_kafka_impl:hosts(BootstrapHosts0), %% Note: this is distinct per node. - ClientID = make_client_id(ResourceId), + ClientID = make_client_id(ResourceId, BridgeType, BridgeName), ClientOpts0 = case Auth of none -> []; @@ -515,11 +517,11 @@ is_dry_run(ResourceId) -> string:equal(TestIdStart, ResourceId) end. --spec make_client_id(resource_id()) -> atom(). -make_client_id(InstanceId) -> - case is_dry_run(InstanceId) of +-spec make_client_id(resource_id(), binary(), atom() | binary()) -> atom(). +make_client_id(ResourceId, BridgeType, BridgeName) -> + case is_dry_run(ResourceId) of false -> - ClientID0 = emqx_bridge_kafka_impl:make_client_id(InstanceId), + ClientID0 = emqx_bridge_kafka_impl:make_client_id(BridgeType, BridgeName), binary_to_atom(ClientID0); true -> %% It is a dry run and we don't want to leak too many diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 0b8f526f6..3485ac752 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -29,10 +29,6 @@ -define(kafka_client_id, kafka_client_id). -define(kafka_producers, kafka_producers). -%% TODO: rename this to `kafka_producer' after alias support is added -%% to hocon; keeping this as just `kafka' for backwards compatibility. --define(BRIDGE_TYPE, kafka). - query_mode(#{kafka := #{query_mode := sync}}) -> simple_sync; query_mode(_) -> @@ -46,6 +42,7 @@ on_start(InstId, Config) -> authentication := Auth, bootstrap_hosts := Hosts0, bridge_name := BridgeName, + bridge_type := BridgeType, connect_timeout := ConnTimeout, kafka := KafkaConfig = #{ message := MessageTemplate, @@ -60,12 +57,11 @@ on_start(InstId, Config) -> KafkaHeadersTokens = preproc_kafka_headers(maps:get(kafka_headers, KafkaConfig, undefined)), KafkaExtHeadersTokens = preproc_ext_headers(maps:get(kafka_ext_headers, KafkaConfig, [])), KafkaHeadersValEncodeMode = maps:get(kafka_header_value_encode_mode, KafkaConfig, none), - BridgeType = ?BRIDGE_TYPE, ResourceId = emqx_bridge_resource:resource_id(BridgeType, BridgeName), ok = emqx_resource:allocate_resource(InstId, ?kafka_resource_id, ResourceId), _ = maybe_install_wolff_telemetry_handlers(ResourceId), Hosts = emqx_bridge_kafka_impl:hosts(Hosts0), - ClientId = emqx_bridge_kafka_impl:make_client_id(InstId), + ClientId = emqx_bridge_kafka_impl:make_client_id(BridgeType, BridgeName), ok = emqx_resource:allocate_resource(InstId, ?kafka_client_id, ClientId), ClientConfig = #{ min_metadata_refresh_interval => MinMetaRefreshInterval, @@ -107,7 +103,7 @@ on_start(InstId, Config) -> _ -> string:equal(TestIdStart, InstId) end, - WolffProducerConfig = producers_config(BridgeName, ClientId, KafkaConfig, IsDryRun), + WolffProducerConfig = producers_config(BridgeType, BridgeName, ClientId, KafkaConfig, IsDryRun), case wolff:ensure_supervised_producers(ClientId, KafkaTopic, WolffProducerConfig) of {ok, Producers} -> ok = emqx_resource:allocate_resource(InstId, ?kafka_producers, Producers), @@ -462,7 +458,7 @@ ssl(#{enable := true} = SSL) -> ssl(_) -> []. -producers_config(BridgeName, ClientId, Input, IsDryRun) -> +producers_config(BridgeType, BridgeName, ClientId, Input, IsDryRun) -> #{ max_batch_bytes := MaxBatchBytes, compression := Compression, @@ -488,10 +484,9 @@ producers_config(BridgeName, ClientId, Input, IsDryRun) -> disk -> {false, replayq_dir(ClientId)}; hybrid -> {true, replayq_dir(ClientId)} end, - BridgeType = ?BRIDGE_TYPE, ResourceID = emqx_bridge_resource:resource_id(BridgeType, BridgeName), #{ - name => make_producer_name(BridgeName, IsDryRun), + name => make_producer_name(BridgeType, BridgeName, IsDryRun), partitioner => partitioner(PartitionStrategy), partition_count_refresh_interval_seconds => PCntRefreshInterval, replayq_dir => ReplayqDir, @@ -516,20 +511,15 @@ replayq_dir(ClientId) -> %% Producer name must be an atom which will be used as a ETS table name for %% partition worker lookup. -make_producer_name(BridgeName, IsDryRun) when is_atom(BridgeName) -> - make_producer_name(atom_to_list(BridgeName), IsDryRun); -make_producer_name(BridgeName, IsDryRun) -> +make_producer_name(_BridgeType, _BridgeName, true = _IsDryRun) -> + %% It is a dry run and we don't want to leak too many atoms + %% so we use the default producer name instead of creating + %% an unique name. + probing_wolff_producers; +make_producer_name(BridgeType, BridgeName, _IsDryRun) -> %% Woff needs an atom for ets table name registration. The assumption here is %% that bridges with new names are not often created. - case IsDryRun of - true -> - %% It is a dry run and we don't want to leak too many atoms - %% so we use the default producer name instead of creating - %% an unique name. - probing_wolff_producers; - false -> - binary_to_atom(iolist_to_binary(["kafka_producer_", BridgeName])) - end. + binary_to_atom(iolist_to_binary([BridgeType, "_", bin(BridgeName)])). with_log_at_error(Fun, Log) -> try diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 756fa8b3b..2f5840f41 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -66,7 +66,7 @@ only_once_tests() -> ]. init_per_suite(Config) -> - Config. + [{bridge_type, <<"kafka_consumer">>} | Config]. end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), @@ -898,8 +898,9 @@ ensure_connected(Config) -> ok. consumer_clientid(Config) -> - ResourceId = resource_id(Config), - binary_to_atom(emqx_bridge_kafka_impl:make_client_id(ResourceId)). + BridgeType = ?config(bridge_type, Config), + KafkaName = ?config(kafka_name, Config), + binary_to_atom(emqx_bridge_kafka_impl:make_client_id(BridgeType, KafkaName)). get_client_connection(Config) -> KafkaHost = ?config(kafka_host, Config), diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index ba11ddf14..432ce8697 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -40,6 +40,7 @@ %% TODO: rename this to `kafka_producer' after alias support is added %% to hocon; keeping this as just `kafka' for backwards compatibility. -define(BRIDGE_TYPE, "kafka"). +-define(BRIDGE_TYPE_BIN, <<"kafka">>). -define(APPS, [emqx_resource, emqx_bridge, emqx_rule_engine, emqx_bridge_kafka]). @@ -438,7 +439,7 @@ t_failed_creation_then_fix(Config) -> {ok, #{config := WrongConfigAtom1}} = emqx_bridge:create( Type, erlang:list_to_atom(Name), WrongConf ), - WrongConfigAtom = WrongConfigAtom1#{bridge_name => Name}, + WrongConfigAtom = WrongConfigAtom1#{bridge_name => Name, bridge_type => ?BRIDGE_TYPE_BIN}, ?assertThrow(Reason when is_list(Reason), ?PRODUCER:on_start(ResourceId, WrongConfigAtom)), %% before throwing, it should cleanup the client process. we %% retry because the supervisor might need some time to really @@ -448,7 +449,7 @@ t_failed_creation_then_fix(Config) -> {ok, #{config := ValidConfigAtom1}} = emqx_bridge:create( Type, erlang:list_to_atom(Name), ValidConf ), - ValidConfigAtom = ValidConfigAtom1#{bridge_name => Name}, + ValidConfigAtom = ValidConfigAtom1#{bridge_name => Name, bridge_type => ?BRIDGE_TYPE_BIN}, {ok, State} = ?PRODUCER:on_start(ResourceId, ValidConfigAtom), Time = erlang:unique_integer(), BinTime = integer_to_binary(Time), @@ -540,7 +541,7 @@ t_nonexistent_topic(_Config) -> {ok, #{config := ValidConfigAtom1}} = emqx_bridge:create( Type, erlang:list_to_atom(Name), Conf ), - ValidConfigAtom = ValidConfigAtom1#{bridge_name => Name}, + ValidConfigAtom = ValidConfigAtom1#{bridge_name => Name, bridge_type => ?BRIDGE_TYPE_BIN}, ?assertThrow(_, ?PRODUCER:on_start(ResourceId, ValidConfigAtom)), ok = emqx_bridge_resource:remove(BridgeId), delete_all_bridges(), @@ -585,7 +586,7 @@ t_send_message_with_headers(Config) -> {ok, #{config := ConfigAtom1}} = emqx_bridge:create( Type, erlang:list_to_atom(Name), Conf ), - ConfigAtom = ConfigAtom1#{bridge_name => Name}, + ConfigAtom = ConfigAtom1#{bridge_name => Name, bridge_type => ?BRIDGE_TYPE_BIN}, {ok, State} = ?PRODUCER:on_start(ResourceId, ConfigAtom), Time1 = erlang:unique_integer(), BinTime1 = integer_to_binary(Time1), @@ -807,7 +808,7 @@ t_wrong_headers_from_message(Config) -> {ok, #{config := ConfigAtom1}} = emqx_bridge:create( Type, erlang:list_to_atom(Name), Conf ), - ConfigAtom = ConfigAtom1#{bridge_name => Name}, + ConfigAtom = ConfigAtom1#{bridge_name => Name, bridge_type => ?BRIDGE_TYPE_BIN}, {ok, State} = ?PRODUCER:on_start(ResourceId, ConfigAtom), Time1 = erlang:unique_integer(), Payload1 = <<"wrong_header">>, From 4dfbc859f958ed7a8ca5ece235908c266d155a7d Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 30 Aug 2023 15:32:58 -0300 Subject: [PATCH 094/119] fix(plugins): update plugin order on whole cluster Fixes https://emqx.atlassian.net/browse/EMQX-10879 --- apps/emqx/test/emqx_cth_cluster.erl | 1 + .../test/emqx_dashboard_api_test_helpers.erl | 12 +- .../src/emqx_mgmt_api_plugins.erl | 2 +- .../test/emqx_mgmt_api_plugins_SUITE.erl | 190 +++++++++++++++++- .../test/emqx_mgmt_api_test_util.erl | 19 +- apps/emqx_plugins/src/emqx_plugins.app.src | 2 +- apps/emqx_plugins/src/emqx_plugins.erl | 43 ++-- apps/emqx_plugins/src/emqx_plugins_cli.erl | 2 +- apps/emqx_plugins/test/emqx_plugins_SUITE.erl | 12 +- changes/ce/fix-11548.en.md | 1 + 10 files changed, 251 insertions(+), 33 deletions(-) create mode 100644 changes/ce/fix-11548.en.md diff --git a/apps/emqx/test/emqx_cth_cluster.erl b/apps/emqx/test/emqx_cth_cluster.erl index 5e8bd4103..194a681d5 100644 --- a/apps/emqx/test/emqx_cth_cluster.erl +++ b/apps/emqx/test/emqx_cth_cluster.erl @@ -20,6 +20,7 @@ -export([stop/1]). -export([share_load_module/2]). +-export([node_name/1]). -define(APPS_CLUSTERING, [gen_rpc, mria, ekka]). diff --git a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl index e233eaaa2..5e8c61e15 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl @@ -28,7 +28,8 @@ multipart_formdata_request/4, host/0, uri/0, - uri/1 + uri/1, + uri/2 ]). -define(HOST, "http://127.0.0.1:18083"). @@ -96,10 +97,15 @@ request(Username, Method, Url, Body) -> host() -> ?HOST. -uri() -> uri([]). +uri() -> + uri([]). + uri(Parts) when is_list(Parts) -> + uri(host(), Parts). + +uri(Host, Parts) when is_list(Host), is_list(Parts) -> NParts = [E || E <- Parts], - host() ++ "/" ++ to_list(filename:join([?BASE_PATH, ?API_VERSION | NParts])). + Host ++ "/" ++ to_list(filename:join([?BASE_PATH, ?API_VERSION | NParts])). auth_header(Username) -> Password = <<"public">>, diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index 3db0c42fb..81c765fc4 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -420,7 +420,7 @@ update_boot_order(post, #{bindings := #{name := Name}, body := Body}) -> {error, Reason} -> {400, #{code => 'BAD_POSITION', message => Reason}}; Position -> - case emqx_plugins:ensure_enabled(Name, Position) of + case emqx_plugins:ensure_enabled(Name, Position, _ConfLocation = global) of ok -> {200}; {error, Reason} -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl index ba613abc4..1cddad76d 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl @@ -19,11 +19,14 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). -define(EMQX_PLUGIN_TEMPLATE_NAME, "emqx_plugin_template"). -define(EMQX_PLUGIN_TEMPLATE_VSN, "5.0.0"). -define(PACKAGE_SUFFIX, ".tar.gz"). +-define(CLUSTER_API_SERVER(PORT), ("http://127.0.0.1:" ++ (integer_to_list(PORT)))). + all() -> emqx_common_test_helpers:all(?MODULE). @@ -48,6 +51,25 @@ end_per_suite(Config) -> emqx_mgmt_api_test_util:end_suite([emqx_plugins, emqx_conf]), ok. +init_per_testcase(t_cluster_update_order = TestCase, Config0) -> + Config = [{api_port, 18085} | Config0], + Cluster = [Node1 | _] = cluster(TestCase, Config), + {ok, API} = init_api(Node1), + [ + {api, API}, + {cluster, Cluster} + | Config + ]; +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(t_cluster_update_order, Config) -> + Cluster = ?config(cluster, Config), + emqx_cth_cluster:stop(Cluster), + ok; +end_per_testcase(_TestCase, _Config) -> + ok. + t_plugins(Config) -> DemoShDir = proplists:get_value(demo_sh_dir, Config), PackagePath = get_demo_plugin_package(DemoShDir), @@ -141,9 +163,83 @@ t_delete_non_existing(_Config) -> ), ok. -list_plugins() -> - Path = emqx_mgmt_api_test_util:api_path(["plugins"]), - case emqx_mgmt_api_test_util:request_api(get, Path) of +t_cluster_update_order(Config) -> + DemoShDir = proplists:get_value(demo_sh_dir, Config), + PackagePath1 = get_demo_plugin_package(DemoShDir), + NameVsn1 = filename:basename(PackagePath1, ?PACKAGE_SUFFIX), + Name2Str = ?EMQX_PLUGIN_TEMPLATE_NAME ++ "_a", + NameVsn2 = Name2Str ++ "-" ++ ?EMQX_PLUGIN_TEMPLATE_VSN, + PackagePath2 = create_renamed_package(PackagePath1, NameVsn2), + Name1 = list_to_binary(?EMQX_PLUGIN_TEMPLATE_NAME), + Name2 = list_to_binary(Name2Str), + + ok = install_plugin(Config, PackagePath1), + ok = install_plugin(Config, PackagePath2), + %% to get them configured... + {ok, _} = update_plugin(Config, NameVsn1, "start"), + {ok, _} = update_plugin(Config, NameVsn2, "start"), + + ?assertMatch( + {ok, [ + #{<<"name">> := Name1}, + #{<<"name">> := Name2} + ]}, + list_plugins(Config) + ), + + ct:pal("moving to rear"), + ?assertMatch({ok, _}, update_boot_order(NameVsn1, #{position => rear}, Config)), + ?assertMatch( + {ok, [ + #{<<"name">> := Name2}, + #{<<"name">> := Name1} + ]}, + list_plugins(Config) + ), + + ct:pal("moving to front"), + ?assertMatch({ok, _}, update_boot_order(NameVsn1, #{position => front}, Config)), + ?assertMatch( + {ok, [ + #{<<"name">> := Name1}, + #{<<"name">> := Name2} + ]}, + list_plugins(Config) + ), + + ct:pal("moving after"), + NameVsn2Bin = list_to_binary(NameVsn2), + ?assertMatch( + {ok, _}, + update_boot_order(NameVsn1, #{position => <<"after:", NameVsn2Bin/binary>>}, Config) + ), + ?assertMatch( + {ok, [ + #{<<"name">> := Name2}, + #{<<"name">> := Name1} + ]}, + list_plugins(Config) + ), + + ct:pal("moving before"), + ?assertMatch( + {ok, _}, + update_boot_order(NameVsn1, #{position => <<"before:", NameVsn2Bin/binary>>}, Config) + ), + ?assertMatch( + {ok, [ + #{<<"name">> := Name1}, + #{<<"name">> := Name2} + ]}, + list_plugins(Config) + ), + + ok. + +list_plugins(Config) -> + #{host := Host, auth := Auth} = get_host_and_auth(Config), + Path = emqx_mgmt_api_test_util:api_path(Host, ["plugins"]), + case emqx_mgmt_api_test_util:request_api(get, Path, Auth) of {ok, Apps} -> {ok, emqx_utils_json:decode(Apps, [return_maps])}; Error -> Error end. @@ -172,16 +268,46 @@ install_plugin(FilePath) -> Error -> Error end. +install_plugin(Config, FilePath) -> + #{host := Host, auth := Auth} = get_host_and_auth(Config), + Path = emqx_mgmt_api_test_util:api_path(Host, ["plugins", "install"]), + case + emqx_mgmt_api_test_util:upload_request( + Path, + FilePath, + "plugin", + <<"application/gzip">>, + [], + Auth + ) + of + {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, <<>>}} -> ok; + Error -> Error + end. + update_plugin(Name, Action) -> Path = emqx_mgmt_api_test_util:api_path(["plugins", Name, Action]), emqx_mgmt_api_test_util:request_api(put, Path). -update_boot_order(Name, MoveBody) -> - Auth = emqx_mgmt_api_test_util:auth_header_(), - Path = emqx_mgmt_api_test_util:api_path(["plugins", Name, "move"]), +update_plugin(Config, Name, Action) when is_list(Config) -> + #{host := Host, auth := Auth} = get_host_and_auth(Config), + Path = emqx_mgmt_api_test_util:api_path(Host, ["plugins", Name, Action]), + emqx_mgmt_api_test_util:request_api(put, Path, Auth). + +update_boot_order(Name, MoveBody, Config) -> + #{host := Host, auth := Auth} = get_host_and_auth(Config), + Path = emqx_mgmt_api_test_util:api_path(Host, ["plugins", Name, "move"]), case emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, MoveBody) of - {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; - Error -> Error + {ok, Res} -> + Resp = + case emqx_utils_json:safe_decode(Res, [return_maps]) of + {ok, Decoded} -> Decoded; + {error, _} -> Res + end, + ct:pal("update_boot_order response:\n ~p", [Resp]), + {ok, Resp}; + Error -> + Error end. uninstall_plugin(Name) -> @@ -218,3 +344,51 @@ update_release_json(["release.json"], FileContent, NewName) -> emqx_utils_json:encode(ContentMap#{<<"name">> => NewName}); update_release_json(_FileName, FileContent, _NewName) -> FileContent. + +cluster(TestCase, Config) -> + APIPort = ?config(api_port, Config), + AppSpecs = app_specs(Config), + Node1Apps = AppSpecs ++ [app_spec_dashboard(APIPort)], + Node2Apps = AppSpecs, + Node1Name = emqx_mgmt_api_plugins_SUITE1, + Node1 = emqx_cth_cluster:node_name(Node1Name), + emqx_cth_cluster:start( + [ + {Node1Name, #{role => core, apps => Node1Apps, join_to => Node1}}, + {emqx_mgmt_api_plugins_SUITE2, #{role => core, apps => Node2Apps, join_to => Node1}} + ], + #{work_dir => filename:join(?config(priv_dir, Config), TestCase)} + ). + +app_specs(_Config) -> + [ + emqx_conf, + emqx, + emqx_management, + emqx_plugins + ]. + +app_spec_dashboard(APIPort) -> + {emqx_dashboard, #{ + config => + #{ + dashboard => + #{ + listeners => + #{ + http => + #{bind => APIPort} + } + } + } + }}. + +init_api(Node) -> + erpc:call(Node, emqx_common_test_http, create_default_app, []). + +get_host_and_auth(Config) when is_list(Config) -> + API = ?config(api, Config), + APIPort = ?config(api_port, Config), + Host = ?CLUSTER_API_SERVER(APIPort), + Auth = emqx_common_test_http:auth_header(API), + #{host => Host, auth => Auth}. diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index d511acf4d..e6c3bb3d6 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -61,6 +61,9 @@ request(Method, Url, Body) -> uri(Parts) -> emqx_dashboard_api_test_helpers:uri(Parts). +uri(Host, Parts) -> + emqx_dashboard_api_test_helpers:uri(Host, Parts). + %% compatible_mode will return as same as 'emqx_dashboard_api_test_helpers:request' request_api_with_body(Method, Url, Body) -> Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, @@ -144,9 +147,15 @@ build_http_header(X) when is_list(X) -> build_http_header(X) -> [X]. +default_server() -> + ?SERVER. + api_path(Parts) -> join_http_path([?SERVER, ?BASE_PATH | Parts]). +api_path(Host, Parts) -> + join_http_path([Host, ?BASE_PATH | Parts]). + api_path_without_base_path(Parts) -> join_http_path([?SERVER | Parts]). @@ -193,9 +202,13 @@ upload_request(URL, FilePath, Name, MimeType, RequestData, AuthorizationToken) - ContentLength = integer_to_list(length(binary_to_list(RequestBody))), Headers = [ {"Content-Length", ContentLength}, - case AuthorizationToken =/= undefined of - true -> {"Authorization", "Bearer " ++ binary_to_list(AuthorizationToken)}; - false -> {} + case AuthorizationToken of + _ when is_tuple(AuthorizationToken) -> + AuthorizationToken; + _ when is_binary(AuthorizationToken) -> + {"Authorization", "Bearer " ++ binary_to_list(AuthorizationToken)}; + _ -> + {} end ], HTTPOptions = [], diff --git a/apps/emqx_plugins/src/emqx_plugins.app.src b/apps/emqx_plugins/src/emqx_plugins.app.src index 368a1ad46..d9c2d50df 100644 --- a/apps/emqx_plugins/src/emqx_plugins.app.src +++ b/apps/emqx_plugins/src/emqx_plugins.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugins, [ {description, "EMQX Plugin Management"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {modules, []}, {mod, {emqx_plugins_app, []}}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 5181000de..0e11062fc 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -29,6 +29,7 @@ ensure_uninstalled/1, ensure_enabled/1, ensure_enabled/2, + ensure_enabled/3, ensure_disabled/1, purge/1, delete_package/1 @@ -240,28 +241,34 @@ ensure_enabled(NameVsn) -> %% @doc Ensure a plugin is enabled at the given position of the plugin list. -spec ensure_enabled(name_vsn(), position()) -> ok | {error, any()}. ensure_enabled(NameVsn, Position) -> - ensure_state(NameVsn, Position, true). + ensure_state(NameVsn, Position, _Enabled = true, _ConfLocation = local). + +-spec ensure_enabled(name_vsn(), position(), local | global) -> ok | {error, any()}. +ensure_enabled(NameVsn, Position, ConfLocation) when + ConfLocation =:= local; ConfLocation =:= global +-> + ensure_state(NameVsn, Position, _Enabled = true, ConfLocation). %% @doc Ensure a plugin is disabled. -spec ensure_disabled(name_vsn()) -> ok | {error, any()}. ensure_disabled(NameVsn) -> - ensure_state(NameVsn, no_move, false). + ensure_state(NameVsn, no_move, false, _ConfLocation = local). -ensure_state(NameVsn, Position, State) when is_binary(NameVsn) -> - ensure_state(binary_to_list(NameVsn), Position, State); -ensure_state(NameVsn, Position, State) -> +ensure_state(NameVsn, Position, State, ConfLocation) when is_binary(NameVsn) -> + ensure_state(binary_to_list(NameVsn), Position, State, ConfLocation); +ensure_state(NameVsn, Position, State, ConfLocation) -> case read_plugin(NameVsn, #{}) of {ok, _} -> Item = #{ name_vsn => NameVsn, enable => State }, - tryit("ensure_state", fun() -> ensure_configured(Item, Position) end); + tryit("ensure_state", fun() -> ensure_configured(Item, Position, ConfLocation) end); {error, Reason} -> {error, Reason} end. -ensure_configured(#{name_vsn := NameVsn} = Item, Position) -> +ensure_configured(#{name_vsn := NameVsn} = Item, Position, ConfLocation) -> Configured = configured(), SplitFun = fun(#{name_vsn := Nv}) -> bin(Nv) =/= bin(NameVsn) end, {Front, Rear} = lists:splitwith(SplitFun, Configured), @@ -274,7 +281,7 @@ ensure_configured(#{name_vsn := NameVsn} = Item, Position) -> [] -> add_new_configured(Configured, Position, Item) end, - ok = put_configured(NewConfigured). + ok = put_configured(NewConfigured, ConfLocation). add_new_configured(Configured, no_move, Item) -> %% default to rear @@ -787,14 +794,23 @@ is_needed_by(AppToStop, RunningApp) -> undefined -> false end. -put_config(Key, Value) when is_atom(Key) -> - put_config([Key], Value); -put_config(Path, Values) when is_list(Path) -> +put_config(Key, Value) -> + put_config(Key, Value, _ConfLocation = local). + +put_config(Key, Value, ConfLocation) when is_atom(Key) -> + put_config([Key], Value, ConfLocation); +put_config(Path, Values, _ConfLocation = local) when is_list(Path) -> Opts = #{rawconf_with_defaults => true, override_to => cluster}, %% Already in cluster_rpc, don't use emqx_conf:update, dead calls case emqx:update_config([?CONF_ROOT | Path], bin_key(Values), Opts) of {ok, _} -> ok; Error -> Error + end; +put_config(Path, Values, _ConfLocation = global) when is_list(Path) -> + Opts = #{rawconf_with_defaults => true, override_to => cluster}, + case emqx_conf:update([?CONF_ROOT | Path], bin_key(Values), Opts) of + {ok, _} -> ok; + Error -> Error end. bin_key(Map) when is_map(Map) -> @@ -812,7 +828,10 @@ get_config(Path, Default) -> install_dir() -> get_config(install_dir, ""). put_configured(Configured) -> - ok = put_config(states, bin_key(Configured)). + put_configured(Configured, _ConfLocation = local). + +put_configured(Configured, ConfLocation) -> + ok = put_config(states, bin_key(Configured), ConfLocation). configured() -> get_config(states, []). diff --git a/apps/emqx_plugins/src/emqx_plugins_cli.erl b/apps/emqx_plugins/src/emqx_plugins_cli.erl index 2cc5f023c..210dedd1c 100644 --- a/apps/emqx_plugins/src/emqx_plugins_cli.erl +++ b/apps/emqx_plugins/src/emqx_plugins_cli.erl @@ -70,7 +70,7 @@ restart(NameVsn, LogFun) -> ?PRINT(emqx_plugins:restart(NameVsn), LogFun). ensure_enabled(NameVsn, Position, LogFun) -> - ?PRINT(emqx_plugins:ensure_enabled(NameVsn, Position), LogFun). + ?PRINT(emqx_plugins:ensure_enabled(NameVsn, Position, _ConfLocation = global), LogFun). ensure_disabled(NameVsn, LogFun) -> ?PRINT(emqx_plugins:ensure_disabled(NameVsn), LogFun). diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl index 9bb3f5e72..bf359374e 100644 --- a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -216,7 +216,7 @@ t_position(Config) -> PosApp2 = <<"position-2">>, ok = write_info_file(Config, PosApp2, FakeInfo), %% fake a disabled plugin in config - ok = emqx_plugins:ensure_state(PosApp2, {before, NameVsn}, false), + ok = ensure_state(PosApp2, {before, NameVsn}, false), ListFun = fun() -> lists:map( fun( @@ -255,14 +255,14 @@ t_start_restart_and_stop(Config) -> Bar2 = <<"bar-2">>, ok = write_info_file(Config, Bar2, FakeInfo), %% fake a disabled plugin in config - ok = emqx_plugins:ensure_state(Bar2, front, false), + ok = ensure_state(Bar2, front, false), assert_app_running(emqx_plugin_template, false), ok = emqx_plugins:ensure_started(), assert_app_running(emqx_plugin_template, true), %% fake enable bar-2 - ok = emqx_plugins:ensure_state(Bar2, rear, true), + ok = ensure_state(Bar2, rear, true), %% should cause an error ?assertError( #{function := _, errors := [_ | _]}, @@ -274,7 +274,7 @@ t_start_restart_and_stop(Config) -> %% stop all ok = emqx_plugins:ensure_stopped(), assert_app_running(emqx_plugin_template, false), - ok = emqx_plugins:ensure_state(Bar2, rear, false), + ok = ensure_state(Bar2, rear, false), ok = emqx_plugins:restart(NameVsn), assert_app_running(emqx_plugin_template, true), @@ -826,3 +826,7 @@ make_tar(Cwd, NameWithVsn, TarfileVsn) -> after file:set_cwd(OriginalCwd) end. + +ensure_state(NameVsn, Position, Enabled) -> + %% NOTE: this is an internal function that is (legacy) exported in test builds only... + emqx_plugins:ensure_state(NameVsn, Position, Enabled, _ConfLocation = local). diff --git a/changes/ce/fix-11548.en.md b/changes/ce/fix-11548.en.md new file mode 100644 index 000000000..3af38310e --- /dev/null +++ b/changes/ce/fix-11548.en.md @@ -0,0 +1 @@ +Fixed an issue that prevented the plugin order to be updated on the whole cluster. From fb094e1d471b6ff51a2b005c1299c16a5cfebb8c Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 30 Aug 2023 20:11:43 +0400 Subject: [PATCH 095/119] test(router): avoid testsuite collisions --- apps/emqx/test/emqx_router_SUITE.erl | 2 +- apps/emqx/test/emqx_router_helper_SUITE.erl | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/apps/emqx/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl index adc898365..c19363f93 100644 --- a/apps/emqx/test/emqx_router_SUITE.erl +++ b/apps/emqx/test/emqx_router_SUITE.erl @@ -40,7 +40,7 @@ groups() -> ]. init_per_group(GroupName, Config) -> - WorkDir = filename:join([?config(priv_dir, Config), GroupName]), + WorkDir = filename:join([?config(priv_dir, Config), ?MODULE, GroupName]), AppSpecs = [ {emqx, #{ config => mk_config(GroupName), diff --git a/apps/emqx/test/emqx_router_helper_SUITE.erl b/apps/emqx/test/emqx_router_helper_SUITE.erl index f4d3cc43a..d65a6f666 100644 --- a/apps/emqx/test/emqx_router_helper_SUITE.erl +++ b/apps/emqx/test/emqx_router_helper_SUITE.erl @@ -40,7 +40,7 @@ groups() -> ]. init_per_group(GroupName, Config) -> - WorkDir = filename:join([?config(priv_dir, Config), GroupName]), + WorkDir = filename:join([?config(priv_dir, Config), ?MODULE, GroupName]), AppSpecs = [{emqx, mk_config(GroupName)}], Apps = emqx_cth_suite:start(AppSpecs, #{work_dir => WorkDir}), [{group_name, GroupName}, {group_apps, Apps} | Config]. @@ -60,16 +60,15 @@ mk_config(routing_schema_v2) -> }. init_per_testcase(TestCase, Config) when - TestCase =:= t_cleanup_membership_mnesia_down; - TestCase =:= t_cleanup_membership_node_down; TestCase =:= t_cleanup_monitor_node_down -> ok = snabbkaffe:start_trace(), - WorkDir = filename:join([?config(priv_dir, Config), ?config(group_name, Config), TestCase]), + GroupName = ?config(group_name, Config), + WorkDir = filename:join([?config(priv_dir, Config), ?MODULE, GroupName, TestCase]), [Slave] = emqx_cth_cluster:start( [ {?MODULE, #{ - apps => [{emqx, mk_config(?config(group_name, Config))}], + apps => [{emqx, mk_config(GroupName)}], join_to => node() }} ], @@ -77,11 +76,10 @@ init_per_testcase(TestCase, Config) when ), [{slave, Slave} | Config]; init_per_testcase(_TestCase, Config) -> + ok = snabbkaffe:start_trace(), Config. end_per_testcase(TestCase, Config) when - TestCase =:= t_cleanup_membership_mnesia_down; - TestCase =:= t_cleanup_membership_node_down; TestCase =:= t_cleanup_monitor_node_down -> Slave = ?config(slave, Config), @@ -89,6 +87,7 @@ end_per_testcase(TestCase, Config) when ok = snabbkaffe:stop(), ok; end_per_testcase(_TestCase, _Config) -> + ok = snabbkaffe:stop(), ok. t_monitor(_) -> From c20ba0572a1efc838007b9db619fa2c3b932bb1e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 31 Aug 2023 15:47:39 +0200 Subject: [PATCH 096/119] chore: bump to 5.2.0-alpha.4 --- apps/emqx/include/emqx_release.hrl | 2 +- deploy/charts/emqx-enterprise/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index c7fa97be7..7b0fcad1a 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.1.5-build.3"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.2.0-alpha.3"). +-define(EMQX_RELEASE_EE, "5.2.0-alpha.4"). %% The HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index 575c6b354..b72b67e81 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.2.0-alpha.3 +version: 5.2.0-alpha.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.2.0-alpha.3 +appVersion: 5.2.0-alpha.4 From 7b607c3300dd2221af4ad39cd77275cc592e9ce2 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Thu, 31 Aug 2023 11:01:49 -0300 Subject: [PATCH 097/119] refactor: remove BridgeName from connector_config callback As `bridge_name` is already in the Config parameter, this callback doesn't need to have it on API anymore. Fixes https://github.com/emqx/emqx/pull/11546/files#r1310778734 --- apps/emqx_bridge/src/emqx_bridge_resource.erl | 17 +++++++---------- .../src/emqx_bridge_azure_event_hub.erl | 4 ++-- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 65e05b950..0756e1382 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -49,11 +49,8 @@ update/4 ]). --callback connector_config(ParsedConfig, BridgeName :: atom() | binary()) -> - ParsedConfig -when - ParsedConfig :: #{atom() => any()}. --optional_callbacks([connector_config/2]). +-callback connector_config(ParsedConfig) -> ParsedConfig when ParsedConfig :: #{atom() => any()}. +-optional_callbacks([connector_config/1]). %% bi-directional bridge with producer/consumer or ingress/egress configs -define(IS_BI_DIR_BRIDGE(TYPE), @@ -391,14 +388,14 @@ parse_confs(Type, Name, Conf) when ?IS_INGRESS_BRIDGE(Type) -> BId = bridge_id(Type, Name), BridgeHookpoint = bridge_hookpoint(BId), Conf#{hookpoint => BridgeHookpoint}; -parse_confs(BridgeType, BridgeName, Config) -> - connector_config(BridgeType, BridgeName, Config). +parse_confs(BridgeType, _BridgeName, Config) -> + connector_config(BridgeType, Config). -connector_config(BridgeType, BridgeName, Config) -> +connector_config(BridgeType, Config) -> Mod = bridge_impl_module(BridgeType), - case erlang:function_exported(Mod, connector_config, 2) of + case erlang:function_exported(Mod, connector_config, 1) of true -> - Mod:connector_config(Config, BridgeName); + Mod:connector_config(Config); false -> Config end. diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl index b34d6132a..abdc6a265 100644 --- a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl @@ -20,7 +20,7 @@ %% emqx_bridge_enterprise "unofficial" API -export([conn_bridge_examples/1]). --export([connector_config/2]). +-export([connector_config/1]). -export([producer_converter/2, host_opts/0]). @@ -166,7 +166,7 @@ values(producer) -> %% `emqx_bridge_resource' API %%------------------------------------------------------------------------------------------------- -connector_config(Config, _BridgeName) -> +connector_config(Config) -> %% Default port for AEH is 9093 BootstrapHosts0 = maps:get(bootstrap_hosts, Config), BootstrapHosts = emqx_schema:parse_servers( From bc0b0fe9e8703dd34cf3e86b27384c7a6881a523 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 31 Aug 2023 10:46:00 -0300 Subject: [PATCH 098/119] fix(nodetool): add patches directory to head of path This enables us to provide patches that affect `call_hocon` from `bin/emqx`. --- bin/nodetool | 8 ++++++++ deploy/packages/deb/debian/rules | 2 ++ deploy/packages/rpm/emqx.spec | 2 ++ 3 files changed, 12 insertions(+) diff --git a/bin/nodetool b/bin/nodetool index 8170e68e2..9838c7813 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -357,6 +357,14 @@ add_libs_dir() -> {error, Reason} -> %% rel file was been deleted by release handler error({failed_to_read_RELEASES_file, RelFile, Reason}) + end, + PatchesDir = filename:join([RootDir, "data", "patches"]), + case filelib:is_dir(PatchesDir) of + true -> + true = code:add_patha(PatchesDir), + ok; + false -> + ok end. add_lib_dir(RootDir, Name, Vsn) -> diff --git a/deploy/packages/deb/debian/rules b/deploy/packages/deb/debian/rules index 4a06bbe26..305b8ffd0 100755 --- a/deploy/packages/deb/debian/rules +++ b/deploy/packages/deb/debian/rules @@ -36,6 +36,8 @@ install: build mkdir -p debian/emqx/usr/lib/emqx/bin mkdir -p debian/emqx/usr/lib/emqx/plugins touch debian/emqx/usr/lib/emqx/plugins/.keep + mkdir -p debian/emqx/usr/lib/emqx/data/patches + touch debian/emqx/usr/lib/emqx/data/patches/.keep mkdir -p debian/emqx/etc/emqx cp bin/* debian/emqx/usr/lib/emqx/bin cp -R lib debian/emqx/usr/lib/emqx diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec index 4839d1a7d..25e82eacd 100644 --- a/deploy/packages/rpm/emqx.spec +++ b/deploy/packages/rpm/emqx.spec @@ -40,6 +40,7 @@ EMQX, a distributed, massively scalable, highly extensible MQTT message broker. %install mkdir -p %{buildroot}%{_lib_home} mkdir -p %{buildroot}%{_lib_home}/plugins +mkdir -p %{buildroot}%{_lib_home}/data/patches mkdir -p %{buildroot}%{_log_dir} mkdir -p %{buildroot}%{_unitdir} mkdir -p %{buildroot}%{_conf_dir} @@ -48,6 +49,7 @@ mkdir -p %{buildroot}%{_var_home} cp -R %{_reldir}/lib %{buildroot}%{_lib_home}/ touch %{buildroot}%{_lib_home}/plugins/.keep +touch %{buildroot}%{_lib_home}/data/patches/.keep cp -R %{_reldir}/erts-* %{buildroot}%{_lib_home}/ cp -R %{_reldir}/releases %{buildroot}%{_lib_home}/ cp -R %{_reldir}/bin %{buildroot}%{_lib_home}/ From 94ca9db2f9590620cf93e73e011276f3fba8b798 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Fri, 1 Sep 2023 14:25:31 +0800 Subject: [PATCH 099/119] chore: upgrade dashboard to e1.2.0-beta.9 for ee --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2f3d7067c..afea4781f 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ endif # Dashboard version # from https://github.com/emqx/emqx-dashboard5 export EMQX_DASHBOARD_VERSION ?= v1.3.2 -export EMQX_EE_DASHBOARD_VERSION ?= e1.2.0-beta.4 +export EMQX_EE_DASHBOARD_VERSION ?= e1.2.0-beta.9 # `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used # In make 4.4+, for backward-compatibility the value from the original environment is used. From eb0385a28f8625f5c6aab3ee6ec49f70ae287b9d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 1 Sep 2023 11:57:30 +0400 Subject: [PATCH 100/119] chore(router): separate test-only function exports --- apps/emqx/src/emqx_router.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 8f2947ba2..092af06da 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -49,8 +49,7 @@ -export([ match_routes/1, - lookup_routes/1, - has_route/2 + lookup_routes/1 ]). -export([print_routes/1]). @@ -62,6 +61,9 @@ -export([topics/0]). +%% Exported for tests +-export([has_route/2]). + %% gen_server callbacks -export([ init/1, From 494626629700ff42613c7a772f1a6b403c6a22c9 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 1 Sep 2023 11:58:07 +0400 Subject: [PATCH 101/119] chore(schema): mention when routing schema v2 is introduced --- rel/i18n/emqx_schema.hocon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 4a8440e83..9e33ffb57 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1552,7 +1552,7 @@ sys_event_messages.desc: broker_routing_storage_schema.desc: """Routing storage schema. Set v1 to leave the default. -Set v2 to enable routing through 2 separate tables, one for topic filter and one for regular topic subscriptions. This schema should increase both subscription and routing performance at the cost of slight increase in memory consumption per subscription. +v2 is introduced in 5.2. It enables routing through 2 separate tables, one for topic filter and one for regular topic subscriptions. This schema should increase both subscription and routing performance at the cost of slight increase in memory consumption per subscription. NOTE: Schema v2 is still experimental. NOTE: Full non-rolling cluster restart is needed after altering this option for it to take any effect.""" From 0a879bffd193ad76602b32c4fea0e60b91636bd6 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 1 Sep 2023 13:47:56 +0400 Subject: [PATCH 102/119] test(router): add e2e testcase for cluster-wide routing Which verifies that both router storage schemas work correctly. --- apps/emqx/test/emqx_cth_cluster.erl | 2 + apps/emqx/test/emqx_routing_SUITE.erl | 148 ++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 apps/emqx/test/emqx_routing_SUITE.erl diff --git a/apps/emqx/test/emqx_cth_cluster.erl b/apps/emqx/test/emqx_cth_cluster.erl index 5e8bd4103..19795d588 100644 --- a/apps/emqx/test/emqx_cth_cluster.erl +++ b/apps/emqx/test/emqx_cth_cluster.erl @@ -237,6 +237,8 @@ default_appspec(emqx_conf, Spec, _NodeSpecs) -> listeners => allocate_listener_ports([tcp, ssl, ws, wss], Spec) } }; +default_appspec(emqx, Spec, _NodeSpecs) -> + #{config => #{listeners => allocate_listener_ports([tcp, ssl, ws, wss], Spec)}}; default_appspec(_App, _, _) -> #{}. diff --git a/apps/emqx/test/emqx_routing_SUITE.erl b/apps/emqx/test/emqx_routing_SUITE.erl new file mode 100644 index 000000000..809158db0 --- /dev/null +++ b/apps/emqx/test/emqx_routing_SUITE.erl @@ -0,0 +1,148 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_routing_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("emqx/include/asserts.hrl"). + +all() -> + [ + {group, routing_schema_v1}, + {group, routing_schema_v2} + ]. + +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + [ + {routing_schema_v1, [], TCs}, + {routing_schema_v2, [], TCs} + ]. + +init_per_group(GroupName, Config) -> + WorkDir = filename:join([?config(priv_dir, Config), ?MODULE, GroupName]), + NodeSpec = #{ + apps => [ + {emqx, #{ + config => mk_config(GroupName), + after_start => fun() -> + % NOTE + % This one is actually defined on `emqx_conf_schema` level, but used + % in `emqx_broker`. Thus we have to resort to this ugly hack. + emqx_config:force_put([rpc, mode], async) + end + }} + ] + }, + NodeSpecs = [ + {emqx_routing_SUITE1, NodeSpec#{role => core}}, + {emqx_routing_SUITE2, NodeSpec#{role => core}}, + {emqx_routing_SUITE3, NodeSpec#{role => replicant}} + ], + Nodes = emqx_cth_cluster:start(NodeSpecs, #{work_dir => WorkDir}), + [{cluster, Nodes}, Config]. + +end_per_group(_GroupName, Config) -> + emqx_cth_cluster:stop(?config(cluster, Config)). + +mk_config(routing_schema_v1) -> + "broker.routing.storage_schema = v1"; +mk_config(routing_schema_v2) -> + "broker.routing.storage_schema = v2". + +t_cluster_routing(Config) -> + Cluster = ?config(cluster, Config), + Clients = [C1, C2, C3] = [start_client(N) || N <- Cluster], + Commands = [ + {fun publish/3, [C1, <<"a/b/c">>, <<"wontsee">>]}, + {fun publish/3, [C2, <<"a/b/d">>, <<"wontsee">>]}, + {fun subscribe/2, [C3, <<"a/+/c/#">>]}, + {fun publish/3, [C1, <<"a/b/c">>, <<"01">>]}, + {fun publish/3, [C2, <<"a/b/d">>, <<"wontsee">>]}, + {fun subscribe/2, [C1, <<"a/b/c">>]}, + {fun subscribe/2, [C2, <<"a/b/+">>]}, + {fun publish/3, [C3, <<"a/b/c">>, <<"02">>]}, + {fun publish/3, [C2, <<"a/b/d">>, <<"03">>]}, + {fun publish/3, [C2, <<"a/b/c/d">>, <<"04">>]}, + {fun subscribe/2, [C3, <<"a/b/d">>]}, + {fun publish/3, [C1, <<"a/b/d">>, <<"05">>]}, + {fun unsubscribe/2, [C3, <<"a/+/c/#">>]}, + {fun publish/3, [C1, <<"a/b/c">>, <<"06">>]}, + {fun publish/3, [C2, <<"a/b/d">>, <<"07">>]}, + {fun publish/3, [C2, <<"a/b/c/d">>, <<"08">>]}, + {fun unsubscribe/2, [C2, <<"a/b/+">>]}, + {fun publish/3, [C1, <<"a/b/c">>, <<"09">>]}, + {fun publish/3, [C2, <<"a/b/d">>, <<"10">>]}, + {fun publish/3, [C2, <<"a/b/c/d">>, <<"11">>]}, + {fun unsubscribe/2, [C3, <<"a/b/d">>]}, + {fun unsubscribe/2, [C1, <<"a/b/c">>]}, + {fun publish/3, [C1, <<"a/b/c">>, <<"wontsee">>]}, + {fun publish/3, [C2, <<"a/b/d">>, <<"wontsee">>]} + ], + ok = lists:foreach(fun({F, Args}) -> erlang:apply(F, Args) end, Commands), + _ = [emqtt:stop(C) || C <- Clients], + Deliveries = ?drainMailbox(), + ?assertMatch( + [ + {pub, C1, #{topic := <<"a/b/c">>, payload := <<"02">>}}, + {pub, C1, #{topic := <<"a/b/c">>, payload := <<"06">>}}, + {pub, C1, #{topic := <<"a/b/c">>, payload := <<"09">>}}, + {pub, C2, #{topic := <<"a/b/c">>, payload := <<"02">>}}, + {pub, C2, #{topic := <<"a/b/d">>, payload := <<"03">>}}, + {pub, C2, #{topic := <<"a/b/d">>, payload := <<"05">>}}, + {pub, C2, #{topic := <<"a/b/c">>, payload := <<"06">>}}, + {pub, C2, #{topic := <<"a/b/d">>, payload := <<"07">>}}, + {pub, C3, #{topic := <<"a/b/c">>, payload := <<"01">>}}, + {pub, C3, #{topic := <<"a/b/c">>, payload := <<"02">>}}, + {pub, C3, #{topic := <<"a/b/c/d">>, payload := <<"04">>}}, + {pub, C3, #{topic := <<"a/b/d">>, payload := <<"05">>}}, + {pub, C3, #{topic := <<"a/b/d">>, payload := <<"07">>}}, + {pub, C3, #{topic := <<"a/b/d">>, payload := <<"10">>}} + ], + lists:sort(Deliveries) + ). + +start_client(Node) -> + Self = self(), + {ok, C} = emqtt:start_link(#{ + port => get_mqtt_tcp_port(Node), + msg_handler => #{ + publish => fun(Msg) -> Self ! {pub, self(), Msg} end + } + }), + {ok, _Props} = emqtt:connect(C), + C. + +publish(C, Topic, Payload) -> + {ok, #{reason_code := 0}} = emqtt:publish(C, Topic, Payload, 1). + +subscribe(C, Topic) -> + % NOTE: sleeping here as lazy way to wait for subscribe to replicate + {ok, _Props, [0]} = emqtt:subscribe(C, Topic), + ok = timer:sleep(200). + +unsubscribe(C, Topic) -> + % NOTE: sleeping here as lazy way to wait for unsubscribe to replicate + {ok, _Props, undefined} = emqtt:unsubscribe(C, Topic), + ok = timer:sleep(200). + +get_mqtt_tcp_port(Node) -> + {_, Port} = erpc:call(Node, emqx_config, get, [[listeners, tcp, default, bind]]), + Port. From 545f1c84a69843a766c10298160e101e88f614d8 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 1 Sep 2023 17:23:34 +0400 Subject: [PATCH 103/119] fix(cth): do not allocate ports for `emqx` app by default This causes tricky and impressively hard to track side effects down the line. Namely, loading `emqx_schema` _after_ `emqx_conf_schema` (as part of cluster node startup sequence) leads to a couple of schema root rewrites, because `emqx_schema` defines similar config roots yet slightly differently (e.g. `authorization`). --- apps/emqx/test/emqx_cth_cluster.erl | 2 - apps/emqx/test/emqx_routing_SUITE.erl | 55 +++++++++++++++++---------- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/apps/emqx/test/emqx_cth_cluster.erl b/apps/emqx/test/emqx_cth_cluster.erl index 19795d588..5e8bd4103 100644 --- a/apps/emqx/test/emqx_cth_cluster.erl +++ b/apps/emqx/test/emqx_cth_cluster.erl @@ -237,8 +237,6 @@ default_appspec(emqx_conf, Spec, _NodeSpecs) -> listeners => allocate_listener_ports([tcp, ssl, ws, wss], Spec) } }; -default_appspec(emqx, Spec, _NodeSpecs) -> - #{config => #{listeners => allocate_listener_ports([tcp, ssl, ws, wss], Spec)}}; default_appspec(_App, _, _) -> #{}. diff --git a/apps/emqx/test/emqx_routing_SUITE.erl b/apps/emqx/test/emqx_routing_SUITE.erl index 809158db0..945e84248 100644 --- a/apps/emqx/test/emqx_routing_SUITE.erl +++ b/apps/emqx/test/emqx_routing_SUITE.erl @@ -38,23 +38,10 @@ groups() -> init_per_group(GroupName, Config) -> WorkDir = filename:join([?config(priv_dir, Config), ?MODULE, GroupName]), - NodeSpec = #{ - apps => [ - {emqx, #{ - config => mk_config(GroupName), - after_start => fun() -> - % NOTE - % This one is actually defined on `emqx_conf_schema` level, but used - % in `emqx_broker`. Thus we have to resort to this ugly hack. - emqx_config:force_put([rpc, mode], async) - end - }} - ] - }, NodeSpecs = [ - {emqx_routing_SUITE1, NodeSpec#{role => core}}, - {emqx_routing_SUITE2, NodeSpec#{role => core}}, - {emqx_routing_SUITE3, NodeSpec#{role => replicant}} + {emqx_routing_SUITE1, #{apps => mk_appspecs(GroupName, 1), role => core}}, + {emqx_routing_SUITE2, #{apps => mk_appspecs(GroupName, 2), role => core}}, + {emqx_routing_SUITE3, #{apps => mk_appspecs(GroupName, 3), role => replicant}} ], Nodes = emqx_cth_cluster:start(NodeSpecs, #{work_dir => WorkDir}), [{cluster, Nodes}, Config]. @@ -62,10 +49,38 @@ init_per_group(GroupName, Config) -> end_per_group(_GroupName, Config) -> emqx_cth_cluster:stop(?config(cluster, Config)). -mk_config(routing_schema_v1) -> - "broker.routing.storage_schema = v1"; -mk_config(routing_schema_v2) -> - "broker.routing.storage_schema = v2". +mk_appspecs(GroupName, N) -> + [ + {emqx, #{ + config => mk_config(GroupName, N), + after_start => fun() -> + % NOTE + % This one is actually defined on `emqx_conf_schema` level, but used + % in `emqx_broker`. Thus we have to resort to this ugly hack. + emqx_config:force_put([rpc, mode], async) + end + }} + ]. + +mk_config(GroupName, N) -> + #{ + broker => mk_config_broker(GroupName), + listeners => mk_config_listeners(N) + }. + +mk_config_broker(routing_schema_v1) -> + #{routing => #{storage_schema => v1}}; +mk_config_broker(routing_schema_v2) -> + #{routing => #{storage_schema => v2}}. + +mk_config_listeners(N) -> + Port = 1883 + N, + #{ + tcp => #{default => #{bind => "127.0.0.1:" ++ integer_to_list(Port)}}, + ssl => #{default => #{enable => false}}, + ws => #{default => #{enable => false}}, + wss => #{default => #{enable => false}} + }. t_cluster_routing(Config) -> Cluster = ?config(cluster, Config), From ebf13c90eac4d024c4918f6ac225f4e115370983 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 1 Sep 2023 19:48:56 +0400 Subject: [PATCH 104/119] test(router): make sort stable in e2e routing testcase --- apps/emqx/test/emqx_routing_SUITE.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_routing_SUITE.erl b/apps/emqx/test/emqx_routing_SUITE.erl index 945e84248..dbc4dc193 100644 --- a/apps/emqx/test/emqx_routing_SUITE.erl +++ b/apps/emqx/test/emqx_routing_SUITE.erl @@ -131,7 +131,12 @@ t_cluster_routing(Config) -> {pub, C3, #{topic := <<"a/b/d">>, payload := <<"07">>}}, {pub, C3, #{topic := <<"a/b/d">>, payload := <<"10">>}} ], - lists:sort(Deliveries) + lists:sort( + fun({pub, CL, #{payload := PL}}, {pub, CR, #{payload := PR}}) -> + {CL, PL} < {CR, PR} + end, + Deliveries + ) ). start_client(Node) -> From 4ab5f8374b88056cb21ad23df5d870e4faaae65b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 1 Sep 2023 19:58:31 +0400 Subject: [PATCH 105/119] chore(schema): mark routing storage schema readOnly Co-authored-by: Thales Macedo Garitezi --- apps/emqx/src/emqx_schema.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index e5f46ffc4..04bd397ec 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1381,6 +1381,7 @@ fields("broker_routing") -> hoconsc:enum([v1, v2]), #{ default => v1, + 'readOnly' => true, desc => ?DESC(broker_routing_storage_schema) } )} From ce571d892ec3dc572be14f083c55f07eba820e10 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 1 Sep 2023 17:50:32 +0800 Subject: [PATCH 106/119] chore: ping opentelemetry v1.3.0-emqx --- apps/emqx_opentelemetry/src/emqx_otel.erl | 2 ++ rebar.config | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/emqx_opentelemetry/src/emqx_otel.erl b/apps/emqx_opentelemetry/src/emqx_otel.erl index 141c8ff04..aa0c7d2d4 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel.erl @@ -73,6 +73,7 @@ setup(_Conf) -> ensure_apps(Conf) -> #{exporter := #{interval := ExporterInterval}} = Conf, {ok, _} = application:ensure_all_started(opentelemetry_exporter), + {ok, _} = application:ensure_all_started(opentelemetry), _ = application:stop(opentelemetry_experimental), ok = application:set_env( opentelemetry_experimental, @@ -92,6 +93,7 @@ ensure_apps(Conf) -> ok. cleanup() -> + _ = application:stop(opentelemetry), _ = application:stop(opentelemetry_experimental), _ = application:stop(opentelemetry_experimental_api), _ = application:stop(opentelemetry_exporter), diff --git a/rebar.config b/rebar.config index 2d605c18f..a9d888261 100644 --- a/rebar.config +++ b/rebar.config @@ -85,13 +85,13 @@ , {jsone, {git, "https://github.com/emqx/jsone.git", {tag, "1.7.1"}}} , {uuid, {git, "https://github.com/okeuday/uuid.git", {tag, "v2.0.6"}}} %% trace - , {opentelemetry_api, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry_api"}} - , {opentelemetry, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry"}} + , {opentelemetry_api, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {tag, "v1.3.0-emqx"}, "apps/opentelemetry_api"}} + , {opentelemetry, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {tag, "v1.3.0-emqx"}, "apps/opentelemetry"}} %% log metrics - , {opentelemetry_experimental, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry_experimental"}} - , {opentelemetry_api_experimental, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry_api_experimental"}} + , {opentelemetry_experimental, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {tag, "v1.3.0-emqx"}, "apps/opentelemetry_experimental"}} + , {opentelemetry_api_experimental, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {tag, "v1.3.0-emqx"}, "apps/opentelemetry_api_experimental"}} %% export - , {opentelemetry_exporter, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry_exporter"}} + , {opentelemetry_exporter, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {tag, "v1.3.0-emqx"}, "apps/opentelemetry_exporter"}} ]}. {xref_ignores, From 0c3db3b9b1e306218d9a9b140c84aa156da3f012 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 4 Sep 2023 09:34:02 +0800 Subject: [PATCH 107/119] chore: ping opentelemetry in mix.exs --- mix.exs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mix.exs b/mix.exs index 3aee37a3d..57322abdf 100644 --- a/mix.exs +++ b/mix.exs @@ -102,26 +102,31 @@ defmodule EMQXUmbrella.MixProject do {:opentelemetry_api, github: "emqx/opentelemetry-erlang", sparse: "apps/opentelemetry_api", + tag: "v1.3.0-emqx", override: true, runtime: false}, {:opentelemetry, github: "emqx/opentelemetry-erlang", sparse: "apps/opentelemetry", + tag: "v1.3.0-emqx", override: true, runtime: false}, {:opentelemetry_api_experimental, github: "emqx/opentelemetry-erlang", sparse: "apps/opentelemetry_api_experimental", + tag: "v1.3.0-emqx", override: true, runtime: false}, {:opentelemetry_experimental, github: "emqx/opentelemetry-erlang", sparse: "apps/opentelemetry_experimental", + tag: "v1.3.0-emqx", override: true, runtime: false}, {:opentelemetry_exporter, github: "emqx/opentelemetry-erlang", sparse: "apps/opentelemetry_exporter", + tag: "v1.3.0-emqx", override: true, runtime: false} ] ++ From d0ccd80850ee83a6648eb9d76855e9a6e9e43c18 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 4 Sep 2023 10:04:14 +0200 Subject: [PATCH 108/119] fix(nodetool): ensure patches dir added for rpm/deb installs --- bin/nodetool | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/nodetool b/bin/nodetool index 9838c7813..a96f5f9fd 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -358,7 +358,10 @@ add_libs_dir() -> %% rel file was been deleted by release handler error({failed_to_read_RELEASES_file, RelFile, Reason}) end, - PatchesDir = filename:join([RootDir, "data", "patches"]), + ok = add_patches_dir(filename:join([RootDir, "data", "patches"])), + ok = add_patches_dir("/var/lib/emqx/patches"). + +add_patches_dir(PatchesDir) -> case filelib:is_dir(PatchesDir) of true -> true = code:add_patha(PatchesDir), From 042c56aa129a77725743e307d71f2c2489f10986 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 4 Sep 2023 10:13:00 +0200 Subject: [PATCH 109/119] fix: revert data dir creation in lib dir --- deploy/packages/deb/debian/rules | 2 -- deploy/packages/rpm/emqx.spec | 2 -- 2 files changed, 4 deletions(-) diff --git a/deploy/packages/deb/debian/rules b/deploy/packages/deb/debian/rules index 305b8ffd0..4a06bbe26 100755 --- a/deploy/packages/deb/debian/rules +++ b/deploy/packages/deb/debian/rules @@ -36,8 +36,6 @@ install: build mkdir -p debian/emqx/usr/lib/emqx/bin mkdir -p debian/emqx/usr/lib/emqx/plugins touch debian/emqx/usr/lib/emqx/plugins/.keep - mkdir -p debian/emqx/usr/lib/emqx/data/patches - touch debian/emqx/usr/lib/emqx/data/patches/.keep mkdir -p debian/emqx/etc/emqx cp bin/* debian/emqx/usr/lib/emqx/bin cp -R lib debian/emqx/usr/lib/emqx diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec index 25e82eacd..4839d1a7d 100644 --- a/deploy/packages/rpm/emqx.spec +++ b/deploy/packages/rpm/emqx.spec @@ -40,7 +40,6 @@ EMQX, a distributed, massively scalable, highly extensible MQTT message broker. %install mkdir -p %{buildroot}%{_lib_home} mkdir -p %{buildroot}%{_lib_home}/plugins -mkdir -p %{buildroot}%{_lib_home}/data/patches mkdir -p %{buildroot}%{_log_dir} mkdir -p %{buildroot}%{_unitdir} mkdir -p %{buildroot}%{_conf_dir} @@ -49,7 +48,6 @@ mkdir -p %{buildroot}%{_var_home} cp -R %{_reldir}/lib %{buildroot}%{_lib_home}/ touch %{buildroot}%{_lib_home}/plugins/.keep -touch %{buildroot}%{_lib_home}/data/patches/.keep cp -R %{_reldir}/erts-* %{buildroot}%{_lib_home}/ cp -R %{_reldir}/releases %{buildroot}%{_lib_home}/ cp -R %{_reldir}/bin %{buildroot}%{_lib_home}/ From 5024304bf9dc83e02567c2cb4114e5486011bfc6 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 4 Sep 2023 14:37:46 +0400 Subject: [PATCH 110/119] fix(router): wait for tables replicate before choosing schema vsn --- apps/emqx/src/emqx_router.erl | 2 + apps/emqx/src/emqx_trie.erl | 5 + apps/emqx/test/emqx_routing_SUITE.erl | 130 ++++++++++++++++++++++---- 3 files changed, 117 insertions(+), 20 deletions(-) diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 092af06da..2cd5ffb96 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -465,6 +465,8 @@ get_schema_vsn() -> -spec init_schema() -> ok. init_schema() -> + ok = mria:wait_for_tables([?ROUTE_TAB, ?ROUTE_TAB_FILTERS]), + ok = emqx_trie:wait_for_tables(), ConfSchema = emqx_config:get([broker, routing, storage_schema]), Schema = choose_schema_vsn(ConfSchema), ok = persistent_term:put(?PT_SCHEMA_VSN, Schema), diff --git a/apps/emqx/src/emqx_trie.erl b/apps/emqx/src/emqx_trie.erl index 229a0e3f4..76be97d3e 100644 --- a/apps/emqx/src/emqx_trie.erl +++ b/apps/emqx/src/emqx_trie.erl @@ -21,6 +21,7 @@ %% Mnesia bootstrap -export([ mnesia/1, + wait_for_tables/0, create_session_trie/1 ]). @@ -105,6 +106,10 @@ create_session_trie(Type) -> ] ). +-spec wait_for_tables() -> ok | {error, _Reason}. +wait_for_tables() -> + mria:wait_for_tables([?TRIE]). + %%-------------------------------------------------------------------- %% Topics APIs %%-------------------------------------------------------------------- diff --git a/apps/emqx/test/emqx_routing_SUITE.erl b/apps/emqx/test/emqx_routing_SUITE.erl index dbc4dc193..6966ac56a 100644 --- a/apps/emqx/test/emqx_routing_SUITE.erl +++ b/apps/emqx/test/emqx_routing_SUITE.erl @@ -26,11 +26,15 @@ all() -> [ {group, routing_schema_v1}, - {group, routing_schema_v2} + {group, routing_schema_v2}, + t_routing_schema_switch_v1, + t_routing_schema_switch_v2 ]. groups() -> - TCs = emqx_common_test_helpers:all(?MODULE), + TCs = [ + t_cluster_routing + ], [ {routing_schema_v1, [], TCs}, {routing_schema_v2, [], TCs} @@ -39,28 +43,38 @@ groups() -> init_per_group(GroupName, Config) -> WorkDir = filename:join([?config(priv_dir, Config), ?MODULE, GroupName]), NodeSpecs = [ - {emqx_routing_SUITE1, #{apps => mk_appspecs(GroupName, 1), role => core}}, - {emqx_routing_SUITE2, #{apps => mk_appspecs(GroupName, 2), role => core}}, - {emqx_routing_SUITE3, #{apps => mk_appspecs(GroupName, 3), role => replicant}} + {emqx_routing_SUITE1, #{apps => [mk_emqx_appspec(GroupName, 1)], role => core}}, + {emqx_routing_SUITE2, #{apps => [mk_emqx_appspec(GroupName, 2)], role => core}}, + {emqx_routing_SUITE3, #{apps => [mk_emqx_appspec(GroupName, 3)], role => replicant}} ], Nodes = emqx_cth_cluster:start(NodeSpecs, #{work_dir => WorkDir}), - [{cluster, Nodes}, Config]. + [{cluster, Nodes} | Config]. end_per_group(_GroupName, Config) -> emqx_cth_cluster:stop(?config(cluster, Config)). -mk_appspecs(GroupName, N) -> - [ - {emqx, #{ - config => mk_config(GroupName, N), - after_start => fun() -> - % NOTE - % This one is actually defined on `emqx_conf_schema` level, but used - % in `emqx_broker`. Thus we have to resort to this ugly hack. - emqx_config:force_put([rpc, mode], async) - end - }} - ]. +init_per_testcase(TC, Config) -> + WorkDir = filename:join([?config(priv_dir, Config), ?MODULE, TC]), + [{work_dir, WorkDir} | Config]. + +end_per_testcase(_TC, _Config) -> + ok. + +mk_emqx_appspec(GroupName, N) -> + {emqx, #{ + config => mk_config(GroupName, N), + after_start => fun() -> + % NOTE + % This one is actually defined on `emqx_conf_schema` level, but used + % in `emqx_broker`. Thus we have to resort to this ugly hack. + emqx_config:force_put([rpc, mode], async) + end + }}. + +mk_genrpc_appspec() -> + {gen_rpc, #{ + override_env => [{port_discovery, stateless}] + }}. mk_config(GroupName, N) -> #{ @@ -68,9 +82,9 @@ mk_config(GroupName, N) -> listeners => mk_config_listeners(N) }. -mk_config_broker(routing_schema_v1) -> +mk_config_broker(Vsn) when Vsn == routing_schema_v1; Vsn == v1 -> #{routing => #{storage_schema => v1}}; -mk_config_broker(routing_schema_v2) -> +mk_config_broker(Vsn) when Vsn == routing_schema_v2; Vsn == v2 -> #{routing => #{storage_schema => v2}}. mk_config_listeners(N) -> @@ -82,6 +96,8 @@ mk_config_listeners(N) -> wss => #{default => #{enable => false}} }. +%% + t_cluster_routing(Config) -> Cluster = ?config(cluster, Config), Clients = [C1, C2, C3] = [start_client(N) || N <- Cluster], @@ -163,6 +179,80 @@ unsubscribe(C, Topic) -> {ok, _Props, undefined} = emqtt:unsubscribe(C, Topic), ok = timer:sleep(200). +%% + +t_routing_schema_switch_v1(Config) -> + t_routing_schema_switch(_From = v2, _To = v1, Config). + +t_routing_schema_switch_v2(Config) -> + t_routing_schema_switch(_From = v1, _To = v2, Config). + +t_routing_schema_switch(VFrom, VTo, Config) -> + % Start first node with routing schema VTo (e.g. v1) + WorkDir = ?config(work_dir, Config), + [Node1] = emqx_cth_cluster:start( + [ + {routing_schema_switch1, #{ + apps => [mk_genrpc_appspec(), mk_emqx_appspec(VTo, 1)] + }} + ], + #{work_dir => WorkDir} + ), + % Ensure there's at least 1 route on Node1 + C1 = start_client(Node1), + ok = subscribe(C1, <<"a/+/c">>), + ok = subscribe(C1, <<"d/e/f/#">>), + % Start rest of nodes with routing schema VFrom (e.g. v2) + [Node2, Node3] = emqx_cth_cluster:start( + [ + {routing_schema_switch2, #{ + apps => [mk_genrpc_appspec(), mk_emqx_appspec(VFrom, 2)], + base_port => 20000, + join_to => Node1 + }}, + {routing_schema_switch3, #{ + apps => [mk_genrpc_appspec(), mk_emqx_appspec(VFrom, 3)], + base_port => 20100, + join_to => Node1 + }} + ], + #{work_dir => WorkDir} + ), + % Verify that new nodes switched to schema v1/v2 in presence of v1/v2 routes respectively + Nodes = [Node1, Node2, Node3], + ?assertEqual( + [{ok, VTo}, {ok, VTo}, {ok, VTo}], + erpc:multicall(Nodes, emqx_router, get_schema_vsn, []) + ), + % Wait for all nodes to agree on cluster state + ?retry( + 500, + 10, + ?assertMatch( + [{ok, [Node1, Node2, Node3]}], + lists:usort(erpc:multicall(Nodes, emqx, running_nodes, [])) + ) + ), + % Verify that routing works as expected + C2 = start_client(Node2), + ok = subscribe(C2, <<"a/+/d">>), + C3 = start_client(Node3), + ok = subscribe(C3, <<"d/e/f/#">>), + {ok, _} = publish(C1, <<"a/b/d">>, <<"hey-newbies">>), + {ok, _} = publish(C2, <<"a/b/c">>, <<"hi">>), + {ok, _} = publish(C3, <<"d/e/f/42">>, <<"hello">>), + ?assertReceive({pub, C2, #{topic := <<"a/b/d">>, payload := <<"hey-newbies">>}}), + ?assertReceive({pub, C1, #{topic := <<"a/b/c">>, payload := <<"hi">>}}), + ?assertReceive({pub, C1, #{topic := <<"d/e/f/42">>, payload := <<"hello">>}}), + ?assertReceive({pub, C3, #{topic := <<"d/e/f/42">>, payload := <<"hello">>}}), + ?assertNotReceive(_), + ok = emqtt:stop(C1), + ok = emqtt:stop(C2), + ok = emqtt:stop(C3), + ok = emqx_cth_cluster:stop(Nodes). + +%% + get_mqtt_tcp_port(Node) -> {_, Port} = erpc:call(Node, emqx_config, get, [[listeners, tcp, default, bind]]), Port. From f0a0c7d4b1b654d5531b5b91236be29a56394b10 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 4 Sep 2023 14:38:52 +0400 Subject: [PATCH 111/119] fix(cthsuite): avoid duplicate application env entries Because `application:set_env/1` complains loudly if there are duplicates. --- apps/emqx/test/emqx_cth_suite.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_cth_suite.erl b/apps/emqx/test/emqx_cth_suite.erl index 307e8d98b..e5668b77a 100644 --- a/apps/emqx/test/emqx_cth_suite.erl +++ b/apps/emqx/test/emqx_cth_suite.erl @@ -261,7 +261,7 @@ merge_envs(false, E2) -> merge_envs(_E, false) -> []; merge_envs(E1, E2) -> - E1 ++ E2. + lists:foldl(fun({K, _} = Opt, EAcc) -> lists:keystore(K, 1, EAcc, Opt) end, E1, E2). merge_config(false, C2) -> C2; From 893f69617a25418f91a0f7e693edec461bdc2cd4 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 4 Sep 2023 14:40:20 +0400 Subject: [PATCH 112/119] chore(test): drop obsolete TODOs --- apps/emqx/test/emqx_router_SUITE.erl | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/apps/emqx/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl index c19363f93..1128112ff 100644 --- a/apps/emqx/test/emqx_router_SUITE.erl +++ b/apps/emqx/test/emqx_router_SUITE.erl @@ -65,24 +65,9 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> clear_tables(). -% t_add_route(_) -> -% error('TODO'). - -% t_do_add_route(_) -> -% error('TODO'). - % t_lookup_routes(_) -> % error('TODO'). -% t_delete_route(_) -> -% error('TODO'). - -% t_do_delete_route(_) -> -% error('TODO'). - -% t_topics(_) -> -% error('TODO'). - t_verify_type(Config) -> case ?config(group_name, Config) of routing_schema_v1 -> From d33ad8a46f8e561cb2889b1689ba8f65dd140f8a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 28 Aug 2023 12:34:35 -0300 Subject: [PATCH 113/119] feat: cluster purge (r5.2) Port of https://github.com/emqx/emqx/pull/11447 targeting `release-52` Fixes https://emqx.atlassian.net/browse/EMQX-10763 --- apps/emqx/priv/bpapi.versions | 6 + apps/emqx/src/emqx_cm.erl | 21 + .../src/emqx_eviction_agent.app.src | 2 +- .../src/emqx_eviction_agent.erl | 62 ++- .../proto/emqx_eviction_agent_proto_v2.erl | 35 ++ .../test/emqx_eviction_agent_test_helpers.erl | 6 +- apps/emqx_modules/src/emqx_delayed.erl | 28 +- .../src/proto/emqx_delayed_proto_v2.erl | 48 +++ apps/emqx_modules/test/emqx_delayed_SUITE.erl | 6 +- .../src/emqx_node_rebalance.app.src | 2 +- .../src/emqx_node_rebalance.erl | 4 +- .../src/emqx_node_rebalance_agent.erl | 24 +- .../src/emqx_node_rebalance_api.erl | 132 ++++++- .../src/emqx_node_rebalance_cli.erl | 80 +++- .../src/emqx_node_rebalance_purge.erl | 233 ++++++++++++ .../src/emqx_node_rebalance_status.erl | 63 ++- .../src/emqx_node_rebalance_sup.erl | 1 + .../emqx_node_rebalance_api_proto_v2.erl | 59 +++ .../proto/emqx_node_rebalance_proto_v2.erl | 84 ++++ .../emqx_node_rebalance_purge_proto_v1.erl | 29 ++ .../emqx_node_rebalance_status_proto_v2.erl | 46 +++ .../test/emqx_node_rebalance_api_SUITE.erl | 153 ++++++-- .../test/emqx_node_rebalance_cli_SUITE.erl | 83 ++++ .../test/emqx_node_rebalance_purge_SUITE.erl | 360 ++++++++++++++++++ .../test/emqx_node_rebalance_status_SUITE.erl | 2 +- changes/ee/feat-11447.en.md | 1 + rel/i18n/emqx_node_rebalance_api.hocon | 18 + 27 files changed, 1521 insertions(+), 67 deletions(-) create mode 100644 apps/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v2.erl create mode 100644 apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_purge.erl create mode 100644 apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_api_proto_v2.erl create mode 100644 apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_proto_v2.erl create mode 100644 apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_purge_proto_v1.erl create mode 100644 apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_status_proto_v2.erl create mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_purge_SUITE.erl create mode 100644 changes/ee/feat-11447.en.md diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index e13f60654..6ef46f477 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -15,7 +15,9 @@ {emqx_conf,3}. {emqx_dashboard,1}. {emqx_delayed,1}. +{emqx_delayed,2}. {emqx_eviction_agent,1}. +{emqx_eviction_agent,2}. {emqx_exhook,1}. {emqx_ft_storage_exporter_fs,1}. {emqx_ft_storage_fs,1}. @@ -37,9 +39,13 @@ {emqx_mgmt_trace,1}. {emqx_mgmt_trace,2}. {emqx_node_rebalance,1}. +{emqx_node_rebalance,2}. {emqx_node_rebalance_api,1}. +{emqx_node_rebalance_api,2}. {emqx_node_rebalance_evacuation,1}. +{emqx_node_rebalance_purge,1}. {emqx_node_rebalance_status,1}. +{emqx_node_rebalance_status,2}. {emqx_persistent_session,1}. {emqx_plugins,1}. {emqx_prometheus,1}. diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index a78ba752e..e3743486c 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -75,6 +75,7 @@ %% Client management -export([ + all_channels_table/1, channel_with_session_table/1, live_connection_table/1 ]). @@ -582,6 +583,26 @@ channel_with_session_table(ConnModuleList) -> sets:is_element(ConnModule, ConnModules) ]). +%% @doc Get clientinfo for all clients, regardless if they use clean start or not. +all_channels_table(ConnModuleList) -> + Ms = ets:fun2ms( + fun({{ClientId, _ChanPid}, Info, _Stats}) -> + {ClientId, Info} + end + ), + Table = ets:table(?CHAN_INFO_TAB, [{traverse, {select, Ms}}]), + ConnModules = sets:from_list(ConnModuleList, [{version, 2}]), + qlc:q([ + {ClientId, ConnState, ConnInfo, ClientInfo} + || {ClientId, #{ + conn_state := ConnState, + clientinfo := ClientInfo, + conninfo := #{conn_mod := ConnModule} = ConnInfo + }} <- + Table, + sets:is_element(ConnModule, ConnModules) + ]). + %% @doc Get all local connection query handle live_connection_table(ConnModules) -> Ms = lists:map(fun live_connection_ms/1, ConnModules), diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src b/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src index f9f6334c3..c11f52fe7 100644 --- a/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src +++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src @@ -1,6 +1,6 @@ {application, emqx_eviction_agent, [ {description, "EMQX Eviction Agent"}, - {vsn, "5.1.0"}, + {vsn, "5.1.1"}, {registered, [ emqx_eviction_agent_sup, emqx_eviction_agent, diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl index 9a29adc69..ab2b9e66a 100644 --- a/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl +++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl @@ -18,14 +18,19 @@ disable/1, status/0, connection_count/0, + all_channels_count/0, session_count/0, session_count/1, evict_connections/1, evict_sessions/2, evict_sessions/3, + purge_sessions/1, evict_session_channel/3 ]). +%% RPC targets +-export([all_local_channels_count/0]). + -behaviour(gen_server). -export([ @@ -113,6 +118,14 @@ evict_sessions(N, Nodes, ConnState) when {error, disabled} end. +purge_sessions(N) -> + case enable_status() of + {enabled, _Kind, _ServerReference} -> + ok = do_purge_sessions(N); + disabled -> + {error, disabled} + end. + %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- @@ -231,6 +244,33 @@ channel_with_session_table(RequiredConnState) -> RequiredConnState =:= ConnState ]). +-spec all_channels_count() -> non_neg_integer(). +all_channels_count() -> + Nodes = emqx:running_nodes(), + Timeout = 15_000, + Results = emqx_eviction_agent_proto_v2:all_channels_count(Nodes, Timeout), + NodeResults = lists:zip(Nodes, Results), + Errors = lists:filter( + fun + ({_Node, {ok, _}}) -> false; + ({_Node, _Err}) -> true + end, + NodeResults + ), + Errors =/= [] andalso + ?SLOG( + warning, + #{ + msg => "error_collecting_all_channels_count", + errors => maps:from_list(Errors) + } + ), + lists:sum([N || {ok, N} <- Results]). + +-spec all_local_channels_count() -> non_neg_integer(). +all_local_channels_count() -> + table_count(emqx_cm:all_channels_table(?CONN_MODULES)). + session_count() -> session_count(any). @@ -247,6 +287,17 @@ take_connections(N) -> ok = qlc:delete_cursor(ChanPidCursor), ChanPids. +take_channels(N) -> + QH = qlc:q([ + {ClientId, ConnInfo, ClientInfo} + || {ClientId, _, ConnInfo, ClientInfo} <- + emqx_cm:all_channels_table(?CONN_MODULES) + ]), + ChanPidCursor = qlc:cursor(QH), + Channels = qlc:next_answers(ChanPidCursor, N), + ok = qlc:delete_cursor(ChanPidCursor), + Channels. + take_channel_with_sessions(N, ConnState) -> ChanPidCursor = qlc:cursor(channel_with_session_table(ConnState)), Channels = qlc:next_answers(ChanPidCursor, N), @@ -283,7 +334,7 @@ evict_session_channel(Nodes, ClientId, ConnInfo, ClientInfo) -> client_info => ClientInfo } ), - case emqx_eviction_agent_proto_v1:evict_session_channel(Node, ClientId, ConnInfo, ClientInfo) of + case emqx_eviction_agent_proto_v2:evict_session_channel(Node, ClientId, ConnInfo, ClientInfo) of {badrpc, Reason} -> ?SLOG( error, @@ -344,5 +395,14 @@ disconnect_channel(ChanPid, ServerReference) -> 'Server-Reference' => ServerReference }}. +do_purge_sessions(N) when N > 0 -> + Channels = take_channels(N), + ok = lists:foreach( + fun({ClientId, _ConnInfo, _ClientInfo}) -> + emqx_cm:discard_session(ClientId) + end, + Channels + ). + select_random(List) when length(List) > 0 -> lists:nth(rand:uniform(length(List)), List). diff --git a/apps/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v2.erl b/apps/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v2.erl new file mode 100644 index 000000000..2d204079c --- /dev/null +++ b/apps/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v2.erl @@ -0,0 +1,35 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_proto_v2). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + evict_session_channel/4, + + %% Introduced in v2: + all_channels_count/2 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.2.1". + +-spec evict_session_channel( + node(), + emqx_types:clientid(), + emqx_types:conninfo(), + emqx_types:clientinfo() +) -> supervisor:startchild_err() | emqx_rpc:badrpc(). +evict_session_channel(Node, ClientId, ConnInfo, ClientInfo) -> + rpc:call(Node, emqx_eviction_agent, evict_session_channel, [ClientId, ConnInfo, ClientInfo]). + +%% Introduced in v2: +-spec all_channels_count([node()], time:time()) -> emqx_rpc:erpc_multicall(non_neg_integer()). +all_channels_count(Nodes, Timeout) -> + erpc:multicall(Nodes, emqx_eviction_agent, all_local_channels_count, [], Timeout). diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl index 7425cb145..130a2628a 100644 --- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl +++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl @@ -9,6 +9,7 @@ emqtt_connect/1, emqtt_connect/2, emqtt_connect_many/2, + emqtt_connect_many/3, stop_many/1, emqtt_try_connect/1, @@ -42,6 +43,9 @@ emqtt_connect(Opts) -> end. emqtt_connect_many(Port, Count) -> + emqtt_connect_many(Port, Count, _StartN = 1). + +emqtt_connect_many(Port, Count, StartN) -> lists:map( fun(N) -> NBin = integer_to_binary(N), @@ -49,7 +53,7 @@ emqtt_connect_many(Port, Count) -> {ok, C} = emqtt_connect([{clientid, ClientId}, {clean_start, false}, {port, Port}]), C end, - lists:seq(1, Count) + lists:seq(StartN, StartN + Count - 1) ). stop_many(Clients) -> diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 559648bdd..22d18c180 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -45,18 +45,22 @@ code_change/3 ]). -%% gen_server callbacks +%% API -export([ load/0, unload/0, load_or_unload/1, get_conf/1, update_config/1, + delayed_count/0, list/1, get_delayed_message/1, get_delayed_message/2, delete_delayed_message/1, delete_delayed_message/2, + clear_all/0, + %% rpc target + clear_all_local/0, cluster_list/1 ]). @@ -167,6 +171,9 @@ unload() -> load_or_unload(Bool) -> gen_server:call(?SERVER, {do_load_or_unload, Bool}). +-spec delayed_count() -> non_neg_integer(). +delayed_count() -> mnesia:table_info(?TAB, size). + list(Params) -> emqx_mgmt_api:paginate(?TAB, Params, ?FORMAT_FUN). @@ -243,7 +250,7 @@ get_delayed_message(Id) -> get_delayed_message(Node, Id) when Node =:= node() -> get_delayed_message(Id); get_delayed_message(Node, Id) -> - emqx_delayed_proto_v1:get_delayed_message(Node, Id). + emqx_delayed_proto_v2:get_delayed_message(Node, Id). -spec delete_delayed_message(binary()) -> with_id_return(). delete_delayed_message(Id) -> @@ -258,7 +265,19 @@ delete_delayed_message(Id) -> delete_delayed_message(Node, Id) when Node =:= node() -> delete_delayed_message(Id); delete_delayed_message(Node, Id) -> - emqx_delayed_proto_v1:delete_delayed_message(Node, Id). + emqx_delayed_proto_v2:delete_delayed_message(Node, Id). + +-spec clear_all() -> ok. +clear_all() -> + Nodes = emqx:running_nodes(), + _ = emqx_delayed_proto_v2:clear_all(Nodes), + ok. + +%% rpc target +-spec clear_all_local() -> ok. +clear_all_local() -> + _ = mria:clear_table(?TAB), + ok. update_config(Config) -> emqx_conf:update([delayed], Config, #{rawconf_with_defaults => true, override_to => cluster}). @@ -408,9 +427,6 @@ do_publish(Key = {Ts, _Id}, Now, Acc) when Ts =< Now -> end, do_publish(mnesia:dirty_next(?TAB, Key), Now, [Key | Acc]). --spec delayed_count() -> non_neg_integer(). -delayed_count() -> mnesia:table_info(?TAB, size). - do_load_or_unload(true, State) -> emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_DELAY_PUB), State; diff --git a/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl b/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl new file mode 100644 index 000000000..6d96dcc66 --- /dev/null +++ b/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl @@ -0,0 +1,48 @@ +%%-------------------------------------------------------------------- +%%Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_delayed_proto_v2). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + get_delayed_message/2, + delete_delayed_message/2, + + %% Introduced in v2: + clear_all/1 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.2.1". + +-spec get_delayed_message(node(), binary()) -> + emqx_delayed:with_id_return(map()) | emqx_rpc:badrpc(). +get_delayed_message(Node, Id) -> + rpc:call(Node, emqx_delayed, get_delayed_message, [Id]). + +-spec delete_delayed_message(node(), binary()) -> emqx_delayed:with_id_return() | emqx_rpc:badrpc(). +delete_delayed_message(Node, Id) -> + rpc:call(Node, emqx_delayed, delete_delayed_message, [Id]). + +%% Introduced in v2: + +-spec clear_all([node()]) -> emqx_rpc:erpc_multicall(ok). +clear_all(Nodes) -> + erpc:multicall(Nodes, emqx_delayed, clear_all_local, []). diff --git a/apps/emqx_modules/test/emqx_delayed_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_SUITE.erl index 56eff8bfc..5085aa2da 100644 --- a/apps/emqx_modules/test/emqx_delayed_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_SUITE.erl @@ -164,15 +164,15 @@ t_cluster(_) -> ?assertMatch( {ok, _}, - emqx_delayed_proto_v1:get_delayed_message(node(), Id) + emqx_delayed_proto_v2:get_delayed_message(node(), Id) ), %% The 'local' and the 'fake-remote' values should be the same, %% however there is a race condition, so we are just assert that they are both 'ok' tuples ?assertMatch({ok, _}, emqx_delayed:get_delayed_message(Id)), - ?assertMatch({ok, _}, emqx_delayed_proto_v1:get_delayed_message(node(), Id)), + ?assertMatch({ok, _}, emqx_delayed_proto_v2:get_delayed_message(node(), Id)), - ok = emqx_delayed_proto_v1:delete_delayed_message(node(), Id), + ok = emqx_delayed_proto_v2:delete_delayed_message(node(), Id), ?assertMatch( {error, _}, diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src index edfa6574e..f6b619d39 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src @@ -1,6 +1,6 @@ {application, emqx_node_rebalance, [ {description, "EMQX Node Rebalance"}, - {vsn, "5.0.4"}, + {vsn, "5.0.5"}, {registered, [ emqx_node_rebalance_sup, emqx_node_rebalance, diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl index 9d53841ed..b2044c5fa 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl @@ -81,7 +81,7 @@ start_link() -> -spec available_nodes(list(node())) -> list(node()). available_nodes(Nodes) when is_list(Nodes) -> - {Available, _} = emqx_node_rebalance_proto_v1:available_nodes(Nodes), + {Available, _} = emqx_node_rebalance_proto_v2:available_nodes(Nodes), lists:filter(fun is_atom/1, Available). %%-------------------------------------------------------------------- @@ -370,7 +370,7 @@ avg(List) when length(List) >= 1 -> lists:sum(List) / length(List). multicall(Nodes, F, A) -> - case apply(emqx_node_rebalance_proto_v1, F, [Nodes | A]) of + case apply(emqx_node_rebalance_proto_v2, F, [Nodes | A]) of {Results, []} -> case lists:partition(fun is_ok/1, lists:zip(Nodes, Results)) of {OkResults, []} -> diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl index 47708d00e..250d03d9c 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl @@ -14,7 +14,9 @@ -export([ start_link/0, enable/1, + enable/2, disable/1, + disable/2, status/0 ]). @@ -40,11 +42,21 @@ start_link() -> -spec enable(pid()) -> ok_or_error(already_enabled | eviction_agent_busy). enable(CoordinatorPid) -> - gen_server:call(?MODULE, {enable, CoordinatorPid}). + enable(CoordinatorPid, ?ENABLE_KIND). + +-spec enable(pid(), emqx_eviction_agent:kind()) -> + ok_or_error(already_enabled | eviction_agent_busy). +enable(CoordinatorPid, Kind) -> + gen_server:call(?MODULE, {enable, CoordinatorPid, Kind}). -spec disable(pid()) -> ok_or_error(already_disabled | invalid_coordinator). disable(CoordinatorPid) -> - gen_server:call(?MODULE, {disable, CoordinatorPid}). + disable(CoordinatorPid, ?ENABLE_KIND). + +-spec disable(pid(), emqx_eviction_agent:kind()) -> + ok_or_error(already_disabled | invalid_coordinator). +disable(CoordinatorPid, Kind) -> + gen_server:call(?MODULE, {disable, CoordinatorPid, Kind}). -spec status() -> status(). status() -> @@ -57,7 +69,7 @@ status() -> init([]) -> {ok, #{}}. -handle_call({enable, CoordinatorPid}, _From, St) -> +handle_call({enable, CoordinatorPid, Kind}, _From, St) -> case St of #{coordinator_pid := _Pid} -> {reply, {error, already_enabled}, St}; @@ -65,7 +77,7 @@ handle_call({enable, CoordinatorPid}, _From, St) -> true = link(CoordinatorPid), EvictionAgentPid = whereis(emqx_eviction_agent), true = link(EvictionAgentPid), - case emqx_eviction_agent:enable(?ENABLE_KIND, undefined) of + case emqx_eviction_agent:enable(Kind, undefined) of ok -> {reply, ok, #{ coordinator_pid => CoordinatorPid, @@ -77,13 +89,13 @@ handle_call({enable, CoordinatorPid}, _From, St) -> {reply, {error, eviction_agent_busy}, St} end end; -handle_call({disable, CoordinatorPid}, _From, St) -> +handle_call({disable, CoordinatorPid, Kind}, _From, St) -> case St of #{ coordinator_pid := CoordinatorPid, eviction_agent_pid := EvictionAgentPid } -> - _ = emqx_eviction_agent:disable(?ENABLE_KIND), + _ = emqx_eviction_agent:disable(Kind), true = unlink(EvictionAgentPid), true = unlink(CoordinatorPid), NewSt = maps:without( diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl index 430ad1e34..a9dc535ad 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl @@ -31,7 +31,9 @@ '/load_rebalance/:node/start'/2, '/load_rebalance/:node/stop'/2, '/load_rebalance/:node/evacuation/start'/2, - '/load_rebalance/:node/evacuation/stop'/2 + '/load_rebalance/:node/evacuation/stop'/2, + '/load_rebalance/:node/purge/start'/2, + '/load_rebalance/:node/purge/stop'/2 ]). %% Schema examples @@ -67,6 +69,9 @@ paths() -> "/load_rebalance/:node/stop", "/load_rebalance/:node/evacuation/start", "/load_rebalance/:node/evacuation/stop" + %% TODO: uncomment after we officially release the feature. + %% "/load_rebalance/:node/purge/start", + %% "/load_rebalance/:node/purge/stop" ]. schema("/load_rebalance/status") -> @@ -176,6 +181,42 @@ schema("/load_rebalance/:node/evacuation/stop") -> } } }. +%% TODO: uncomment after we officially release the feature. +%% schema("/load_rebalance/:node/purge/start") -> +%% #{ +%% 'operationId' => '/load_rebalance/:node/purge/start', +%% post => #{ +%% tags => [<<"load_rebalance">>], +%% summary => <<"Start purge on the whole cluster">>, +%% description => ?DESC("cluster_purge_start"), +%% parameters => [param_node()], +%% 'requestBody' => +%% emqx_dashboard_swagger:schema_with_examples( +%% ref(purge_start), +%% purge_example() +%% ), +%% responses => #{ +%% 200 => response_schema(), +%% 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), +%% 404 => error_codes([?NOT_FOUND], <<"Not Found">>) +%% } +%% } +%% }; +%% schema("/load_rebalance/:node/purge/stop") -> +%% #{ +%% 'operationId' => '/load_rebalance/:node/purge/stop', +%% post => #{ +%% tags => [<<"load_rebalance">>], +%% summary => <<"Stop purge on the whole cluster">>, +%% description => ?DESC("cluster_purge_stop"), +%% parameters => [param_node()], +%% responses => #{ +%% 200 => response_schema(), +%% 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), +%% 404 => error_codes([?NOT_FOUND], <<"Not Found">>) +%% } +%% } +%% }. %%-------------------------------------------------------------------- %% Handlers @@ -188,16 +229,20 @@ schema("/load_rebalance/:node/evacuation/stop") -> {rebalance, Stats} -> {200, format_status(rebalance, Stats)}; {evacuation, Stats} -> - {200, format_status(evacuation, Stats)} + {200, format_status(evacuation, Stats)}; + {purge, Stats} -> + {200, format_status(purge, Stats)} end. '/load_rebalance/global_status'(get, #{}) -> #{ evacuations := Evacuations, + purges := Purges, rebalances := Rebalances } = emqx_node_rebalance_status:global_status(), {200, #{ evacuations => format_as_map_list(Evacuations), + purges => format_as_map_list(Purges), rebalances => format_as_map_list(Rebalances) }}. @@ -214,7 +259,7 @@ schema("/load_rebalance/:node/evacuation/stop") -> Params1 = translate(rebalance_start, Params0), with_nodes_at_key(nodes, Params1, fun(Params2) -> wrap_rpc( - Node, emqx_node_rebalance_api_proto_v1:node_rebalance_start(Node, Params2) + Node, emqx_node_rebalance_api_proto_v2:node_rebalance_start(Node, Params2) ) end) end). @@ -222,7 +267,7 @@ schema("/load_rebalance/:node/evacuation/stop") -> '/load_rebalance/:node/stop'(post, #{bindings := #{node := NodeBin}}) -> emqx_utils_api:with_node(NodeBin, fun(Node) -> wrap_rpc( - Node, emqx_node_rebalance_api_proto_v1:node_rebalance_stop(Node) + Node, emqx_node_rebalance_api_proto_v2:node_rebalance_stop(Node) ) end). @@ -234,7 +279,7 @@ schema("/load_rebalance/:node/evacuation/stop") -> with_nodes_at_key(migrate_to, Params1, fun(Params2) -> wrap_rpc( Node, - emqx_node_rebalance_api_proto_v1:node_rebalance_evacuation_start( + emqx_node_rebalance_api_proto_v2:node_rebalance_evacuation_start( Node, Params2 ) ) @@ -244,7 +289,27 @@ schema("/load_rebalance/:node/evacuation/stop") -> '/load_rebalance/:node/evacuation/stop'(post, #{bindings := #{node := NodeBin}}) -> emqx_utils_api:with_node(NodeBin, fun(Node) -> wrap_rpc( - Node, emqx_node_rebalance_api_proto_v1:node_rebalance_evacuation_stop(Node) + Node, emqx_node_rebalance_api_proto_v2:node_rebalance_evacuation_stop(Node) + ) + end). + +'/load_rebalance/:node/purge/start'(post, #{ + bindings := #{node := NodeBin}, body := Params0 +}) -> + emqx_utils_api:with_node(NodeBin, fun(Node) -> + Params1 = translate(purge_start, Params0), + wrap_rpc( + Node, + emqx_node_rebalance_api_proto_v2:node_rebalance_purge_start( + Node, Params1 + ) + ) + end). + +'/load_rebalance/:node/purge/stop'(post, #{bindings := #{node := NodeBin}}) -> + emqx_utils_api:with_node(NodeBin, fun(Node) -> + wrap_rpc( + Node, emqx_node_rebalance_api_proto_v2:node_rebalance_purge_stop(Node) ) end). @@ -483,6 +548,17 @@ fields(rebalance_evacuation_start) -> } )} ]; +fields(purge_start) -> + [ + {"purge_rate", + mk( + pos_integer(), + #{ + desc => ?DESC(purge_rate), + required => false + } + )} + ]; fields(local_status_disabled) -> [ {"status", @@ -687,6 +763,38 @@ fields(global_evacuation_status) -> } )} ]; +fields(global_purge_status) -> + without( + [ + "status", + "process", + "connection_eviction_rate", + "session_eviction_rate", + "connection_goal", + "disconnected_session_goal", + "session_recipients", + "recipients" + ], + fields(local_status_enabled) + ) ++ + [ + {"purge_rate", + mk( + pos_integer(), + #{ + desc => ?DESC(local_status_purge_rate), + required => false + } + )}, + {"node", + mk( + binary(), + #{ + desc => ?DESC(evacuation_status_node), + required => true + } + )} + ]; fields(global_status) -> [ {"evacuations", @@ -697,6 +805,14 @@ fields(global_status) -> required => false } )}, + {"purges", + mk( + hoconsc:array(ref(global_purge_status)), + #{ + desc => ?DESC(global_status_purges), + required => false + } + )}, {"rebalances", mk( hoconsc:array(ref(global_coordinator_status)), @@ -735,6 +851,10 @@ rebalance_evacuation_example() -> } }. +%% TODO: uncomment after we officially release the feature. +%% purge_example() -> +%% #{purge => #{purge_rate => 100}}. + local_status_response_schema() -> hoconsc:union([ref(local_status_disabled), ref(local_status_enabled)]). diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl index 66f7a1789..9e0a173ea 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl @@ -29,6 +29,15 @@ cli(["start" | StartArgs]) -> emqx_ctl:print("Rebalance is already enabled~n"), false end; + {purge, Opts} -> + case emqx_node_rebalance_purge:start(Opts) of + ok -> + emqx_ctl:print("Rebalance(purge) started~n"), + true; + {error, Reason} -> + emqx_ctl:print("Rebalance(purge) start error: ~p~n", [Reason]), + false + end; {rebalance, Opts} -> case emqx_node_rebalance:start(Opts) of ok -> @@ -55,6 +64,7 @@ cli(["node-status"]) -> cli(["status"]) -> #{ evacuations := Evacuations, + purges := Purges, rebalances := Rebalances } = emqx_node_rebalance_status:global_status(), lists:foreach( @@ -69,6 +79,18 @@ cli(["status"]) -> end, Evacuations ), + lists:foreach( + fun({Node, Status}) -> + emqx_ctl:print( + "--------------------------------------------------------------------~n" + ), + emqx_ctl:print( + "Node ~p: purge~n~s", + [Node, emqx_node_rebalance_status:format_local_status(Status)] + ) + end, + Purges + ), lists:foreach( fun({Node, Status}) -> emqx_ctl:print( @@ -82,10 +104,14 @@ cli(["status"]) -> Rebalances ); cli(["stop"]) -> - case emqx_node_rebalance_evacuation:status() of - {enabled, _} -> - ok = emqx_node_rebalance_evacuation:stop(), - emqx_ctl:print("Rebalance(evacuation) stopped~n"), + Checks = + [ + {evacuation, fun emqx_node_rebalance_evacuation:status/0, + fun emqx_node_rebalance_evacuation:stop/0}, + {purge, fun emqx_node_rebalance_purge:status/0, fun emqx_node_rebalance_purge:stop/0} + ], + case do_stop(Checks) of + ok -> true; disabled -> case emqx_node_rebalance:status() of @@ -112,6 +138,13 @@ cli(_) -> "Start current node evacuation with optional server redirect to the specified servers" }, + %% TODO: uncomment after we officially release the feature. + %% { + %% "rebalance start --purge \\\n" + %% " [--purge-rate CountPerSec]", + %% "Start purge on all running nodes in the cluster" + %% }, + { "rebalance start \\\n" " [--nodes \"node1@host1 node2@host2\"] \\\n" @@ -140,7 +173,11 @@ cli(_) -> node_status(NodeStatus) -> case NodeStatus of - {Process, Status} when Process =:= evacuation orelse Process =:= rebalance -> + {Process, Status} when + Process =:= evacuation; + Process =:= purge; + Process =:= rebalance + -> emqx_ctl:print( "Rebalance type: ~p~n~s~n", [Process, emqx_node_rebalance_status:format_local_status(Status)] @@ -160,6 +197,13 @@ start_args(Args) -> {error, _} = Error -> Error end; + {ok, #{"--purge" := true} = Collected} -> + case validate_purge(maps:to_list(Collected), #{}) of + {ok, Validated} -> + {purge, Validated}; + {error, _} = Error -> + Error + end; {ok, #{} = Collected} -> case validate_rebalance(maps:to_list(Collected), #{}) of {ok, Validated} -> @@ -180,6 +224,11 @@ collect_args(["--redirect-to", ServerReference | Args], Map) -> collect_args(Args, Map#{"--redirect-to" => ServerReference}); collect_args(["--migrate-to", MigrateTo | Args], Map) -> collect_args(Args, Map#{"--migrate-to" => MigrateTo}); +%% purge +collect_args(["--purge" | Args], Map) -> + collect_args(Args, Map#{"--purge" => true}); +collect_args(["--purge-rate", PurgeRate | Args], Map) -> + collect_args(Args, Map#{"--purge-rate" => PurgeRate}); %% rebalance collect_args(["--nodes", Nodes | Args], Map) -> collect_args(Args, Map#{"--nodes" => Nodes}); @@ -239,6 +288,15 @@ validate_evacuation([{"--migrate-to", MigrateTo} | Rest], Map) -> validate_evacuation(Rest, _Map) -> {error, io_lib:format("unknown evacuation arguments: ~p", [Rest])}. +validate_purge([], Map) -> + {ok, Map}; +validate_purge([{"--purge", _} | Rest], Map) -> + validate_purge(Rest, Map); +validate_purge([{"--purge-rate", _} | _] = Opts, Map) -> + validate_pos_int(purge_rate, Opts, Map, fun validate_purge/2); +validate_purge(Rest, _Map) -> + {error, io_lib:format("unknown purge arguments: ~p", [Rest])}. + validate_rebalance([], Map) -> {ok, Map}; validate_rebalance([{"--wait-health-check", _} | _] = Opts, Map) -> @@ -306,3 +364,15 @@ strings_to_atoms([Str | Rest], Atoms, Invalid) -> {error, _} -> strings_to_atoms(Rest, Atoms, [Str | Invalid]) end. + +do_stop([{Type, Check, Stop} | Rest]) -> + case Check() of + {enabled, _} -> + ok = Stop(), + emqx_ctl:print("Rebalance(~s) stopped~n", [Type]), + ok; + disabled -> + do_stop(Rest) + end; +do_stop([]) -> + disabled. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_purge.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_purge.erl new file mode 100644 index 000000000..81f1bfe03 --- /dev/null +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_purge.erl @@ -0,0 +1,233 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_purge). + +-include("emqx_node_rebalance.hrl"). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-export([ + start/1, + status/0, + stop/0 +]). + +-export([start_link/0]). + +-behaviour(gen_statem). + +-export([ + init/1, + callback_mode/0, + handle_event/4, + code_change/4 +]). + +-export_type([ + start_opts/0, + start_error/0, + stop_error/0 +]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +-define(DEFAULT_PURGE_RATE, 500). +-define(ENABLE_KIND, purge). + +%% gen_statem states +-define(disabled, disabled). +-define(purging, purging). +-define(cleaning_data, cleaning_data). + +-type start_opts() :: #{ + purge_rate => pos_integer() +}. +-type start_error() :: already_started. +-type stop_error() :: not_started. +-type stats() :: #{ + initial_sessions := non_neg_integer(), + current_sessions := non_neg_integer(), + purge_rate := pos_integer() +}. +-type status() :: {enabled, stats()} | disabled. + +-spec start(start_opts()) -> ok_or_error(start_error()). +start(StartOpts) -> + Opts = maps:merge(default_opts(), StartOpts), + gen_statem:call(?MODULE, {start, Opts}). + +-spec stop() -> ok_or_error(not_started). +stop() -> + gen_statem:call(?MODULE, stop). + +-spec status() -> status(). +status() -> + gen_statem:call(?MODULE, status). + +-spec start_link() -> startlink_ret(). +start_link() -> + gen_statem:start_link({local, ?MODULE}, ?MODULE, [], []). + +%%-------------------------------------------------------------------- +%% gen_statem callbacks +%%-------------------------------------------------------------------- + +callback_mode() -> handle_event_function. + +%% states: disabled, purging, cleaning_data + +init([]) -> + {ok, disabled, #{}}. + +%% start +handle_event({call, From}, {start, Opts}, ?disabled, #{} = Data) -> + ok = enable_purge(), + ?SLOG(warning, #{ + msg => "cluster_purge_started", + opts => Opts + }), + NewData = init_data(Data, Opts), + {next_state, ?purging, NewData, [ + {state_timeout, 0, purge}, + {reply, From, ok} + ]}; +handle_event({call, From}, {start, _Opts}, _State, #{}) -> + {keep_state_and_data, [{reply, From, {error, already_started}}]}; +%% stop +handle_event({call, From}, stop, ?disabled, #{}) -> + {keep_state_and_data, [{reply, From, {error, not_started}}]}; +handle_event({call, From}, stop, _State, Data) -> + ok = disable_purge(), + ?SLOG(warning, #{msg => "cluster_purge_stopped"}), + {next_state, disabled, deinit(Data), [{reply, From, ok}]}; +%% status +handle_event({call, From}, status, ?disabled, #{}) -> + {keep_state_and_data, [{reply, From, disabled}]}; +handle_event({call, From}, status, State, Data) -> + Stats = maps:with( + [ + initial_sessions, + current_sessions, + purge_rate + ], + Data + ), + {keep_state_and_data, [ + {reply, From, {enabled, Stats#{state => State}}} + ]}; +%% session purge +handle_event( + state_timeout, + purge, + ?purging, + #{ + purge_rate := PurgeRate + } = Data +) -> + case emqx_eviction_agent:all_channels_count() of + Sessions when Sessions > 0 -> + ok = purge_sessions(PurgeRate), + ?tp( + warning, + "cluster_purge_evict_sessions", + #{ + count => Sessions, + purge_rate => PurgeRate + } + ), + NewData = Data#{current_sessions => Sessions}, + {keep_state, NewData, [{state_timeout, ?EVICT_INTERVAL, purge}]}; + _Sessions = 0 -> + NewData = Data#{current_conns => 0}, + ?SLOG(warning, #{msg => "cluster_purge_evict_sessions_done"}), + {next_state, ?cleaning_data, NewData, [ + {state_timeout, 0, clean_retained_messages} + ]} + end; +handle_event( + state_timeout, + clean_retained_messages, + ?cleaning_data, + Data +) -> + ?SLOG(warning, #{msg => "cluster_purge_cleaning_data"}), + ok = emqx_retainer:clean(), + ok = emqx_delayed:clear_all(), + ?tp(warning, "cluster_purge_done", #{}), + ok = disable_purge(), + ?tp(warning, "cluster_purge_finished_successfully", #{}), + {next_state, ?disabled, deinit(Data)}; +handle_event({call, From}, Msg, State, Data) -> + ?SLOG(warning, #{msg => "unknown_call", call => Msg, state => State, data => Data}), + {keep_state_and_data, [{reply, From, ignored}]}; +handle_event(info, Msg, State, Data) -> + ?SLOG(warning, #{msg => "unknown_info", info => Msg, state => State, data => Data}), + keep_state_and_data; +handle_event(cast, Msg, State, Data) -> + ?SLOG(warning, #{msg => "unknown_cast", cast => Msg, state => State, data => Data}), + keep_state_and_data. + +code_change(_Vsn, State, Data, _Extra) -> + {ok, State, Data}. + +%%-------------------------------------------------------------------- +%% internal funs +%%-------------------------------------------------------------------- + +default_opts() -> + #{ + purge_rate => ?DEFAULT_PURGE_RATE + }. + +init_data(Data0, Opts) -> + Data1 = maps:merge(Data0, Opts), + SessCount = emqx_eviction_agent:session_count(), + Data1#{ + initial_sessions => SessCount, + current_sessions => SessCount + }. + +deinit(Data) -> + Keys = + [initial_sessions, current_sessions | maps:keys(default_opts())], + maps:without(Keys, Data). + +multicall(Nodes, F, A) -> + case apply(emqx_node_rebalance_proto_v2, F, [Nodes | A]) of + {Results, []} -> + case lists:partition(fun is_ok/1, lists:zip(Nodes, Results)) of + {_OkResults, []} -> + ok; + {_, BadResults} -> + %% we crash on errors so that the coordinator death is signalled to + %% the eviction agents in the cluster. + error({bad_nodes, BadResults}) + end; + {_, [_BadNode | _] = BadNodes} -> + error({bad_nodes, BadNodes}) + end. + +is_ok({_Node, {ok, _}}) -> true; +is_ok({_Node, ok}) -> true; +is_ok(_) -> false. + +enable_purge() -> + Nodes = emqx:running_nodes(), + _ = multicall(Nodes, enable_rebalance_agent, [self(), ?ENABLE_KIND]), + ok. + +disable_purge() -> + Nodes = emqx:running_nodes(), + _ = multicall(Nodes, disable_rebalance_agent, [self(), ?ENABLE_KIND]), + ok. + +purge_sessions(PurgeRate) -> + Nodes = emqx:running_nodes(), + _ = multicall(Nodes, purge_sessions, [PurgeRate]), + ok. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl index a0102c4f4..dbeb4d97f 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl @@ -15,6 +15,7 @@ %% For RPC -export([ evacuation_status/0, + purge_status/0, rebalance_status/0 ]). @@ -22,11 +23,13 @@ %% APIs %%-------------------------------------------------------------------- --spec local_status() -> disabled | {evacuation, map()} | {rebalance, map()}. +-spec local_status() -> disabled | {evacuation, map()} | {purge, map()} | {rebalance, map()}. local_status() -> - case emqx_node_rebalance_evacuation:status() of - {enabled, Status} -> - {evacuation, evacuation(Status)}; + Checks = [ + {evacuation, fun emqx_node_rebalance_evacuation:status/0, fun evacuation/1}, + {purge, fun emqx_node_rebalance_purge:status/0, fun purge/1} + ], + case do_local_status(Checks) of disabled -> case emqx_node_rebalance_agent:status() of {enabled, CoordinatorPid} -> @@ -38,28 +41,37 @@ local_status() -> end; disabled -> disabled - end + end; + Res -> + Res end. --spec local_status(node()) -> disabled | {evacuation, map()} | {rebalance, map()}. +-spec local_status(node()) -> disabled | {evacuation, map()} | {purge, map()} | {rebalance, map()}. local_status(Node) -> - emqx_node_rebalance_status_proto_v1:local_status(Node). + emqx_node_rebalance_status_proto_v2:local_status(Node). -spec format_local_status(map()) -> iodata(). format_local_status(Status) -> format_status(Status, local_status_field_format_order()). --spec global_status() -> #{rebalances := [{node(), map()}], evacuations := [{node(), map()}]}. +-spec global_status() -> + #{ + rebalances := [{node(), map()}], + evacuations := [{node(), map()}], + purges := [{node(), map()}] + }. global_status() -> Nodes = emqx:running_nodes(), - {RebalanceResults, _} = emqx_node_rebalance_status_proto_v1:rebalance_status(Nodes), + {RebalanceResults, _} = emqx_node_rebalance_status_proto_v2:rebalance_status(Nodes), Rebalances = [ {Node, coordinator_rebalance(Status)} || {Node, {enabled, Status}} <- RebalanceResults ], - {EvacuatioResults, _} = emqx_node_rebalance_status_proto_v1:evacuation_status(Nodes), - Evacuations = [{Node, evacuation(Status)} || {Node, {enabled, Status}} <- EvacuatioResults], - #{rebalances => Rebalances, evacuations => Evacuations}. + {EvacuationResults, _} = emqx_node_rebalance_status_proto_v2:evacuation_status(Nodes), + Evacuations = [{Node, evacuation(Status)} || {Node, {enabled, Status}} <- EvacuationResults], + {PurgeResults, _} = emqx_node_rebalance_status_proto_v2:purge_status(Nodes), + Purges = [{Node, purge(Status)} || {Node, {enabled, Status}} <- PurgeResults], + #{rebalances => Rebalances, evacuations => Evacuations, purges => Purges}. -spec format_coordinator_status(map()) -> iodata(). format_coordinator_status(Status) -> @@ -85,6 +97,17 @@ evacuation(Status) -> } }. +purge(Status) -> + #{ + state => maps:get(state, Status), + purge_rate => maps:get(purge_rate, Status), + session_goal => 0, + stats => #{ + initial_sessions => maps:get(initial_sessions, Status), + current_sessions => maps:get(current_sessions, Status) + } + }. + local_rebalance(#{donors := Donors} = Stats, Node) -> case lists:member(Node, Donors) of true -> {rebalance, donor_rebalance(Stats, Node)}; @@ -159,6 +182,7 @@ local_status_field_format_order() -> coordinator_node, connection_eviction_rate, session_eviction_rate, + purge_rate, connection_goal, session_goal, disconnected_session_goal, @@ -201,6 +225,8 @@ format_local_status_field({connection_eviction_rate, ConnEvictRate}) -> io_lib:format("Connection eviction rate: ~p connections/second~n", [ConnEvictRate]); format_local_status_field({session_eviction_rate, SessEvictRate}) -> io_lib:format("Session eviction rate: ~p sessions/second~n", [SessEvictRate]); +format_local_status_field({purge_rate, PurgeRate}) -> + io_lib:format("Purge rate: ~p sessions/second~n", [PurgeRate]); format_local_status_field({connection_goal, ConnGoal}) -> io_lib:format("Connection goal: ~p~n", [ConnGoal]); format_local_status_field({session_goal, SessGoal}) -> @@ -231,8 +257,21 @@ format_local_stats(Stats) -> ) ]. +do_local_status([{Type, Get, Cont} | Rest]) -> + case Get() of + disabled -> + do_local_status(Rest); + {enabled, Status} -> + {Type, Cont(Status)} + end; +do_local_status([]) -> + disabled. + evacuation_status() -> {node(), emqx_node_rebalance_evacuation:status()}. +purge_status() -> + {node(), emqx_node_rebalance_purge:status()}. + rebalance_status() -> {node(), emqx_node_rebalance:status()}. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl index cfaccc4c2..8ec16c1e7 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl @@ -15,6 +15,7 @@ start_link() -> init([]) -> Childs = [ + child_spec(emqx_node_rebalance_purge, []), child_spec(emqx_node_rebalance_evacuation, []), child_spec(emqx_node_rebalance_agent, []), child_spec(emqx_node_rebalance, []) diff --git a/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_api_proto_v2.erl b/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_api_proto_v2.erl new file mode 100644 index 000000000..2b5b4bca3 --- /dev/null +++ b/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_api_proto_v2.erl @@ -0,0 +1,59 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_api_proto_v2). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + node_rebalance_evacuation_start/2, + node_rebalance_evacuation_stop/1, + + node_rebalance_start/2, + node_rebalance_stop/1, + + %% Introduced in v2: + node_rebalance_purge_start/2, + node_rebalance_purge_stop/1 +]). + +-include_lib("emqx/include/bpapi.hrl"). +-include_lib("emqx/include/types.hrl"). + +introduced_in() -> + "5.2.1". + +-spec node_rebalance_evacuation_start(node(), emqx_node_rebalance_evacuation:start_opts()) -> + emqx_rpc:badrpc() | ok_or_error(emqx_node_rebalance_evacuation:start_error()). +node_rebalance_evacuation_start(Node, #{} = Opts) -> + rpc:call(Node, emqx_node_rebalance_evacuation, start, [Opts]). + +-spec node_rebalance_evacuation_stop(node()) -> + emqx_rpc:badrpc() | ok_or_error(not_started). +node_rebalance_evacuation_stop(Node) -> + rpc:call(Node, emqx_node_rebalance_evacuation, stop, []). + +-spec node_rebalance_start(node(), emqx_node_rebalance:start_opts()) -> + emqx_rpc:badrpc() | ok_or_error(emqx_node_rebalance:start_error()). +node_rebalance_start(Node, Opts) -> + rpc:call(Node, emqx_node_rebalance, start, [Opts]). + +-spec node_rebalance_stop(node()) -> + emqx_rpc:badrpc() | ok_or_error(not_started). +node_rebalance_stop(Node) -> + rpc:call(Node, emqx_node_rebalance, stop, []). + +%% Introduced in v2: + +-spec node_rebalance_purge_start(node(), emqx_node_rebalance_purge:start_opts()) -> + emqx_rpc:badrpc() | ok_or_error(emqx_node_rebalance_purge:start_error()). +node_rebalance_purge_start(Node, #{} = Opts) -> + rpc:call(Node, emqx_node_rebalance_purge, start, [Opts]). + +-spec node_rebalance_purge_stop(node()) -> + emqx_rpc:badrpc() | ok_or_error(emqx_node_rebalance_purge:stop_error()). +node_rebalance_purge_stop(Node) -> + rpc:call(Node, emqx_node_rebalance_purge, stop, []). diff --git a/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_proto_v2.erl b/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_proto_v2.erl new file mode 100644 index 000000000..ca8233288 --- /dev/null +++ b/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_proto_v2.erl @@ -0,0 +1,84 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_proto_v2). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + available_nodes/1, + evict_connections/2, + evict_sessions/4, + connection_counts/1, + session_counts/1, + enable_rebalance_agent/2, + disable_rebalance_agent/2, + disconnected_session_counts/1, + + %% Introduced in v2: + enable_rebalance_agent/3, + disable_rebalance_agent/3, + purge_sessions/2 +]). + +-include_lib("emqx/include/bpapi.hrl"). +-include_lib("emqx/include/types.hrl"). + +introduced_in() -> + "5.2.1". + +-spec available_nodes([node()]) -> emqx_rpc:multicall_result(node()). +available_nodes(Nodes) -> + rpc:multicall(Nodes, emqx_node_rebalance, is_node_available, []). + +-spec evict_connections([node()], non_neg_integer()) -> + emqx_rpc:multicall_result(ok_or_error(disabled)). +evict_connections(Nodes, Count) -> + rpc:multicall(Nodes, emqx_eviction_agent, evict_connections, [Count]). + +-spec evict_sessions([node()], non_neg_integer(), [node()], emqx_channel:conn_state()) -> + emqx_rpc:multicall_result(ok_or_error(disabled)). +evict_sessions(Nodes, Count, RecipientNodes, ConnState) -> + rpc:multicall(Nodes, emqx_eviction_agent, evict_sessions, [Count, RecipientNodes, ConnState]). + +-spec connection_counts([node()]) -> emqx_rpc:multicall_result({ok, non_neg_integer()}). +connection_counts(Nodes) -> + rpc:multicall(Nodes, emqx_node_rebalance, connection_count, []). + +-spec session_counts([node()]) -> emqx_rpc:multicall_result({ok, non_neg_integer()}). +session_counts(Nodes) -> + rpc:multicall(Nodes, emqx_node_rebalance, session_count, []). + +-spec enable_rebalance_agent([node()], pid()) -> + emqx_rpc:multicall_result(ok_or_error(already_enabled | eviction_agent_busy)). +enable_rebalance_agent(Nodes, OwnerPid) -> + rpc:multicall(Nodes, emqx_node_rebalance_agent, enable, [OwnerPid]). + +-spec disable_rebalance_agent([node()], pid()) -> + emqx_rpc:multicall_result(ok_or_error(already_disabled | invalid_coordinator)). +disable_rebalance_agent(Nodes, OwnerPid) -> + rpc:multicall(Nodes, emqx_node_rebalance_agent, disable, [OwnerPid]). + +-spec disconnected_session_counts([node()]) -> emqx_rpc:multicall_result({ok, non_neg_integer()}). +disconnected_session_counts(Nodes) -> + rpc:multicall(Nodes, emqx_node_rebalance, disconnected_session_count, []). + +%% Introduced in v2: + +-spec enable_rebalance_agent([node()], pid(), emqx_eviction_agent:kind()) -> + emqx_rpc:multicall_result(ok_or_error(already_enabled | eviction_agent_busy)). +enable_rebalance_agent(Nodes, OwnerPid, Kind) -> + rpc:multicall(Nodes, emqx_node_rebalance_agent, enable, [OwnerPid, Kind]). + +-spec disable_rebalance_agent([node()], pid(), emqx_eviction_agent:kind()) -> + emqx_rpc:multicall_result(ok_or_error(already_disabled | invalid_coordinator)). +disable_rebalance_agent(Nodes, OwnerPid, Kind) -> + rpc:multicall(Nodes, emqx_node_rebalance_agent, disable, [OwnerPid, Kind]). + +-spec purge_sessions([node()], non_neg_integer()) -> + emqx_rpc:multicall_result(ok_or_error(disabled)). +purge_sessions(Nodes, Count) -> + rpc:multicall(Nodes, emqx_eviction_agent, purge_sessions, [Count]). diff --git a/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_purge_proto_v1.erl b/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_purge_proto_v1.erl new file mode 100644 index 000000000..8b2b63bc4 --- /dev/null +++ b/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_purge_proto_v1.erl @@ -0,0 +1,29 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_purge_proto_v1). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + start/2, + stop/1 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.2.1". + +-spec start([node()], emqx_node_rebalance_purge:start_opts()) -> + emqx_rpc:erpc_multicall(ok | {error, emqx_node_rebalance_purge:start_error()}). +start(Nodes, Opts) -> + erpc:multicall(Nodes, emqx_node_rebalance_purge, start, [Opts]). + +-spec stop([node()]) -> + emqx_rpc:erpc_multicall(ok | {error, emqx_node_rebalance_purge:stop_error()}). +stop(Nodes) -> + erpc:multicall(Nodes, emqx_node_rebalance_purge, stop, []). diff --git a/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_status_proto_v2.erl b/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_status_proto_v2.erl new file mode 100644 index 000000000..af8981b59 --- /dev/null +++ b/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_status_proto_v2.erl @@ -0,0 +1,46 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_status_proto_v2). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + local_status/1, + rebalance_status/1, + evacuation_status/1, + + %% Introduced in v2: + purge_status/1 +]). + +-include_lib("emqx/include/bpapi.hrl"). +-include_lib("emqx/include/types.hrl"). + +introduced_in() -> + "5.2.1". + +-spec local_status(node()) -> + emqx_rpc:badrpc() | disabled | {evacuation, map()} | {rebalance, map()}. +local_status(Node) -> + rpc:call(Node, emqx_node_rebalance_status, local_status, []). + +-spec rebalance_status([node()]) -> + emqx_rpc:multicall_result({node(), map()}). +rebalance_status(Nodes) -> + rpc:multicall(Nodes, emqx_node_rebalance_status, rebalance_status, []). + +-spec evacuation_status([node()]) -> + emqx_rpc:multicall_result({node(), map()}). +evacuation_status(Nodes) -> + rpc:multicall(Nodes, emqx_node_rebalance_status, evacuation_status, []). + +%% Introduced in v2: + +-spec purge_status([node()]) -> + emqx_rpc:multicall_result({node(), map()}). +purge_status(Nodes) -> + rpc:multicall(Nodes, emqx_node_rebalance_status, purge_status, []). diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl index 188e6bf71..017e85971 100644 --- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl +++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl @@ -38,32 +38,35 @@ end_per_suite(_Config) -> ok. init_per_testcase(Case, Config) -> - [{DonorNode, _} | _] = - ClusterNodes = emqx_eviction_agent_test_helpers:start_cluster( - [ - {case_specific_node_name(?MODULE, Case, '_donor'), 2883}, - {case_specific_node_name(?MODULE, Case, '_recipient'), 3883} - ], - ?START_APPS, - [{emqx, data_dir, case_specific_data_dir(Case, Config)}] + DonorNode = case_specific_node_name(?MODULE, Case, '_donor'), + RecipientNode = case_specific_node_name(?MODULE, Case, '_recipient'), + Spec = #{ + role => core, + join_to => emqx_cth_cluster:node_name(DonorNode), + listeners => true, + apps => app_specs() + }, + Cluster = [{Node, Spec} || Node <- [DonorNode, RecipientNode]], + ClusterNodes = + [Node1 | _] = emqx_cth_cluster:start( + Cluster, + #{work_dir => ?config(priv_dir, Config)} ), - - ok = rpc:call(DonorNode, emqx_mgmt_api_test_util, init_suite, []), - ok = take_auth_header_from(DonorNode), - + ok = rpc:call(Node1, emqx_mgmt_api_test_util, init_suite, []), + ok = take_auth_header_from(Node1), [{cluster_nodes, ClusterNodes} | Config]. end_per_testcase(_Case, Config) -> - _ = emqx_eviction_agent_test_helpers:stop_cluster( - ?config(cluster_nodes, Config), - ?START_APPS - ). + Nodes = ?config(cluster_nodes, Config), + erpc:multicall(Nodes, meck, unload, []), + _ = emqx_cth_cluster:stop(Nodes), + ok. %%-------------------------------------------------------------------- %% Tests %%-------------------------------------------------------------------- t_start_evacuation_validation(Config) -> - [{DonorNode, _}, {RecipientNode, _}] = ?config(cluster_nodes, Config), + [DonorNode, RecipientNode] = ?config(cluster_nodes, Config), BadOpts = [ #{conn_evict_rate => <<"conn">>}, #{sess_evict_rate => <<"sess">>}, @@ -117,10 +120,87 @@ t_start_evacuation_validation(Config) -> api_get(["load_rebalance", "global_status"]) ). +%% TODO: uncomment after we officially release the feature. +skipped_t_start_purge_validation(Config) -> + [Node1 | _] = ?config(cluster_nodes, Config), + Port1 = get_mqtt_port(Node1, tcp), + BadOpts = [ + #{purge_rate => <<"conn">>}, + #{purge_rate => 0}, + #{purge_rate => -1}, + #{purge_rate => 1.1}, + #{unknown => <<"Value">>} + ], + lists:foreach( + fun(Opts) -> + ?assertMatch( + {ok, 400, #{}}, + api_post( + ["load_rebalance", atom_to_list(Node1), "purge", "start"], + Opts + ), + Opts + ) + end, + BadOpts + ), + ?assertMatch( + {ok, 404, #{}}, + api_post( + ["load_rebalance", "bad@node", "purge", "start"], + #{} + ) + ), + + process_flag(trap_exit, true), + Conns = emqtt_connect_many(Port1, 100), + + ?assertMatch( + {ok, 200, #{}}, + api_post( + ["load_rebalance", atom_to_list(Node1), "purge", "start"], + #{purge_rate => 10} + ) + ), + + Node1Bin = atom_to_binary(Node1), + ?assertMatch( + {ok, 200, #{<<"purges">> := [#{<<"node">> := Node1Bin}]}}, + api_get(["load_rebalance", "global_status"]) + ), + + ?assertMatch( + {ok, 200, #{ + <<"process">> := <<"purge">>, + <<"purge_rate">> := 10, + <<"session_goal">> := 0, + <<"state">> := <<"purging">>, + <<"stats">> := + #{ + <<"current_sessions">> := _, + <<"initial_sessions">> := 100 + } + }}, + api_get(["load_rebalance", "status"]) + ), + + ?assertMatch( + {ok, 200, #{}}, + api_post( + ["load_rebalance", atom_to_list(Node1), "purge", "stop"], + #{} + ) + ), + + ok = stop_many(Conns), + + ok. + t_start_rebalance_validation(Config) -> process_flag(trap_exit, true), - [{DonorNode, DonorPort}, {RecipientNode, _}] = ?config(cluster_nodes, Config), + [DonorNode, RecipientNode] = ?config(cluster_nodes, Config), + DonorPort = get_mqtt_port(DonorNode, tcp), BadOpts = [ #{conn_evict_rate => <<"conn">>}, @@ -189,7 +269,7 @@ t_start_rebalance_validation(Config) -> ok = stop_many(Conns). t_start_stop_evacuation(Config) -> - [{DonorNode, _}, {RecipientNode, _}] = ?config(cluster_nodes, Config), + [DonorNode, RecipientNode] = ?config(cluster_nodes, Config), StartOpts = maps:merge( maps:get(evacuation, emqx_node_rebalance_api:rebalance_evacuation_example()), @@ -284,7 +364,8 @@ t_start_stop_evacuation(Config) -> t_start_stop_rebalance(Config) -> process_flag(trap_exit, true), - [{DonorNode, DonorPort}, {RecipientNode, _}] = ?config(cluster_nodes, Config), + [DonorNode, RecipientNode] = ?config(cluster_nodes, Config), + DonorPort = get_mqtt_port(DonorNode, tcp), ?assertMatch( {ok, 200, #{<<"status">> := <<"disabled">>}}, @@ -390,7 +471,7 @@ t_start_stop_rebalance(Config) -> ok = stop_many(Conns). t_availability_check(Config) -> - [{DonorNode, _} | _] = ?config(cluster_nodes, Config), + [DonorNode | _] = ?config(cluster_nodes, Config), ?assertMatch( {ok, 200, #{}}, api_get(["load_rebalance", "availability_check"]) @@ -425,7 +506,12 @@ api_get(Path) -> api_post(Path, Data) -> case request(post, uri(Path), Data) of {ok, Code, ResponseBody} -> - {ok, Code, jiffy:decode(ResponseBody, [return_maps])}; + Res = + case emqx_utils_json:safe_decode(ResponseBody, [return_maps]) of + {ok, Decoded} -> Decoded; + {error, _} -> ResponseBody + end, + {ok, Code, Res}; {error, _} = Error -> Error end. @@ -444,3 +530,26 @@ case_specific_data_dir(Case, Config) -> undefined -> undefined; PrivDir -> filename:join(PrivDir, atom_to_list(Case)) end. + +app_specs() -> + [ + {emqx, #{ + before_start => fun() -> + emqx_app:set_config_loader(?MODULE) + end, + override_env => [{boot_modules, [broker, listeners]}] + }}, + {emqx_retainer, #{ + config => + #{ + retainer => + #{enable => true} + } + }}, + emqx_eviction_agent, + emqx_node_rebalance + ]. + +get_mqtt_port(Node, Type) -> + {_IP, Port} = erpc:call(Node, emqx_config, get, [[listeners, Type, default, bind]]), + Port. diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl index 54ecad026..7d0cab0ce 100644 --- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl +++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl @@ -156,6 +156,80 @@ t_evacuation(_Config) -> emqx_node_rebalance_evacuation:status() ). +t_purge(_Config) -> + %% start with invalid args + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--purge", "--foo-bar"]) + ), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--purge", "--purge-rate", "foobar"]) + ), + + %% not used by this scenario + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--purge", "--conn-evict-rate", "1"]) + ), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--purge", "--sess-evict-rate", "1"]) + ), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--purge", "--wait-takeover", "1"]) + ), + + ?assertNot( + emqx_node_rebalance_cli:cli([ + "start", + "--purge", + "--migrate-to", + atom_to_list(node()) + ]) + ), + with_some_sessions(fun() -> + ?assert( + emqx_node_rebalance_cli:cli([ + "start", + "--purge", + "--purge-rate", + "10" + ]) + ), + + %% status + ok = emqx_node_rebalance_cli:cli(["status"]), + ok = emqx_node_rebalance_cli:cli(["node-status"]), + ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]), + + ?assertMatch( + {enabled, #{}}, + emqx_node_rebalance_purge:status() + ), + + %% already enabled + ?assertNot( + emqx_node_rebalance_cli:cli([ + "start", + "--purge", + "--purge-rate", + "10" + ]) + ), + true = emqx_node_rebalance_cli:cli(["stop"]), + ok + end), + %% stop + + false = emqx_node_rebalance_cli:cli(["stop"]), + + ?assertEqual( + disabled, + emqx_node_rebalance_purge:status() + ), + + ok. + t_rebalance(Config) -> process_flag(trap_exit, true), @@ -289,3 +363,12 @@ emqx_node_rebalance_cli(Node, Args) -> Result -> Result end. + +%% to avoid it finishing too fast +with_some_sessions(Fn) -> + emqx_common_test_helpers:with_mock( + emqx_eviction_agent, + all_channels_count, + fun() -> 100 end, + Fn + ). diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_purge_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_purge_SUITE.erl new file mode 100644 index 000000000..7cdcc4d71 --- /dev/null +++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_purge_SUITE.erl @@ -0,0 +1,360 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_purge_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/asserts.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-import( + emqx_eviction_agent_test_helpers, + [ + emqtt_connect/1, + emqtt_try_connect/1, + case_specific_node_name/3 + ] +). + +all() -> + [{group, one_node}, {group, two_nodes}]. + +groups() -> + [ + {one_node, [], one_node_cases()}, + {two_nodes, [], two_nodes_cases()} + ]. + +two_nodes_cases() -> + [ + t_already_started_two, + t_session_purged + ]. + +one_node_cases() -> + emqx_common_test_helpers:all(?MODULE) -- two_nodes_cases(). + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([]), + ok. + +init_per_group(one_node, Config) -> + [{cluster_type, one_node} | Config]; +init_per_group(two_nodes, Config) -> + [{cluster_type, two_nodes} | Config]. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(TestCase, Config) -> + ct:timetrap({seconds, 30}), + Nodes = + [Node1 | _] = + case ?config(cluster_type, Config) of + one_node -> + [case_specific_node_name(?MODULE, TestCase, '_1')]; + two_nodes -> + [ + case_specific_node_name(?MODULE, TestCase, '_1'), + case_specific_node_name(?MODULE, TestCase, '_2') + ] + end, + Spec = #{ + role => core, + join_to => emqx_cth_cluster:node_name(Node1), + listeners => true, + apps => app_specs() + }, + Cluster = [{Node, Spec} || Node <- Nodes], + ClusterNodes = emqx_cth_cluster:start( + Cluster, + #{work_dir => ?config(priv_dir, Config)} + ), + ok = snabbkaffe:start_trace(), + [{cluster_nodes, ClusterNodes} | Config]. + +end_per_testcase(_TestCase, Config) -> + Nodes = ?config(cluster_nodes, Config), + ok = snabbkaffe:stop(), + erpc:multicall(Nodes, meck, unload, []), + ok = emqx_cth_cluster:stop(Nodes), + ok. + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +app_specs() -> + [ + {emqx, #{ + before_start => fun() -> + emqx_app:set_config_loader(?MODULE) + end, + override_env => [{boot_modules, [broker, listeners]}] + }}, + {emqx_retainer, #{ + config => + #{ + retainer => + #{enable => true} + } + }}, + {emqx_modules, #{ + config => + #{delayed => #{enable => true}} + }}, + emqx_eviction_agent, + emqx_node_rebalance + ]. + +opts(_Config) -> + #{ + purge_rate => 10 + }. + +case_specific_data_dir(Case, Config) -> + case ?config(priv_dir, Config) of + undefined -> undefined; + PrivDir -> filename:join(PrivDir, atom_to_list(Case)) + end. + +get_mqtt_port(Node, Type) -> + {_IP, Port} = erpc:call(Node, emqx_config, get, [[listeners, Type, default, bind]]), + Port. + +%% to avoid it finishing too fast +with_some_sessions(Node, Fn) -> + erpc:call(Node, fun() -> + emqx_common_test_helpers:with_mock( + emqx_eviction_agent, + all_channels_count, + fun() -> 100 end, + Fn + ) + end). + +drain_exits([ClientPid | Rest]) -> + receive + {'EXIT', ClientPid, _Reason} -> + drain_exits(Rest) + after 1_000 -> + ct:pal("mailbox:\n ~p", [process_info(self(), messages)]), + ct:fail("pid ~p didn't die", [ClientPid]) + end; +drain_exits([]) -> + ok. + +emqtt_connect_many(Port, Count) -> + emqtt_connect_many(Port, Count, _StartN = 1). + +%% start many clients with mixed clean_start flags +emqtt_connect_many(Port, Count, StartN) -> + lists:map( + fun(N) -> + NBin = integer_to_binary(N), + ClientId = <<"client-", NBin/binary>>, + CleanStart = N rem 2 == 0, + {ok, C} = emqtt_connect([{clientid, ClientId}, {clean_start, CleanStart}, {port, Port}]), + C + end, + lists:seq(StartN, StartN + Count - 1) + ). + +%%-------------------------------------------------------------------- +%% Test Cases : one node +%%-------------------------------------------------------------------- + +t_agent_busy(Config) -> + [Node] = ?config(cluster_nodes, Config), + + ok = rpc:call(Node, emqx_eviction_agent, enable, [other_rebalance, undefined]), + + erpc:call(Node, fun() -> + ?assertExit( + {{{bad_nodes, [{Node, {error, eviction_agent_busy}}]}, _}, _}, + emqx_node_rebalance_purge:start(opts(Config)) + ) + end), + + ok. + +t_already_started(Config) -> + [Node] = ?config(cluster_nodes, Config), + with_some_sessions(Node, fun() -> + ok = emqx_node_rebalance_purge:start(opts(Config)), + + ?assertEqual( + {error, already_started}, + emqx_node_rebalance_purge:start(opts(Config)) + ), + + ?assertEqual( + ok, + emqx_node_rebalance_purge:stop() + ), + + ok + end), + ok. + +t_not_started(Config) -> + [Node] = ?config(cluster_nodes, Config), + + ?assertEqual( + {error, not_started}, + rpc:call(Node, emqx_node_rebalance_purge, stop, []) + ). + +t_start(Config) -> + [Node] = ?config(cluster_nodes, Config), + Port = get_mqtt_port(Node, tcp), + + with_some_sessions(Node, fun() -> + process_flag(trap_exit, true), + ok = snabbkaffe:start_trace(), + + ?assertEqual( + ok, + emqx_node_rebalance_purge:start(opts(Config)) + ), + ?assertEqual({error, {use_another_server, #{}}}, emqtt_try_connect([{port, Port}])), + ok + end), + ok. + +t_non_persistence(Config) -> + [Node] = ?config(cluster_nodes, Config), + Port = get_mqtt_port(Node, tcp), + + %% to avoid it finishing too fast + with_some_sessions(Node, fun() -> + process_flag(trap_exit, true), + ok = snabbkaffe:start_trace(), + + ?assertEqual( + ok, + emqx_node_rebalance_purge:start(opts(Config)) + ), + + ?assertMatch( + {error, {use_another_server, #{}}}, + emqtt_try_connect([{port, Port}]) + ), + + ok = supervisor:terminate_child(emqx_node_rebalance_sup, emqx_node_rebalance_purge), + {ok, _} = supervisor:restart_child(emqx_node_rebalance_sup, emqx_node_rebalance_purge), + + ?assertMatch( + ok, + emqtt_try_connect([{port, Port}]) + ), + ?assertMatch(disabled, emqx_node_rebalance_purge:status()), + ok + end), + ok. + +t_unknown_messages(Config) -> + process_flag(trap_exit, true), + + [Node] = ?config(cluster_nodes, Config), + + ok = rpc:call(Node, emqx_node_rebalance_purge, start, [opts(Config)]), + Pid = rpc:call(Node, erlang, whereis, [emqx_node_rebalance_purge]), + Pid ! unknown, + ok = gen_server:cast(Pid, unknown), + ?assertEqual( + ignored, + gen_server:call(Pid, unknown) + ), + + ok. + +%%-------------------------------------------------------------------- +%% Test Cases : two nodes +%%-------------------------------------------------------------------- + +t_already_started_two(Config) -> + [Node1, _Node2] = ?config(cluster_nodes, Config), + with_some_sessions(Node1, fun() -> + ok = emqx_node_rebalance_purge:start(opts(Config)), + + ?assertEqual( + {error, already_started}, + emqx_node_rebalance_purge:start(opts(Config)) + ), + + ?assertEqual( + ok, + emqx_node_rebalance_purge:stop() + ), + + ok + end), + ?assertEqual( + {error, not_started}, + rpc:call(Node1, emqx_node_rebalance_purge, stop, []) + ), + + ok. + +t_session_purged(Config) -> + process_flag(trap_exit, true), + + [Node1, Node2] = ?config(cluster_nodes, Config), + Port1 = get_mqtt_port(Node1, tcp), + Port2 = get_mqtt_port(Node2, tcp), + + %% N.B.: it's important to have an asymmetric number of clients for this test, as + %% otherwise the scenario might happen to finish successfully due to the wrong + %% reasons! + NumClientsNode1 = 5, + NumClientsNode2 = 35, + Node1Clients = emqtt_connect_many(Port1, NumClientsNode1, _StartN1 = 1), + Node2Clients = emqtt_connect_many(Port2, NumClientsNode2, _StartN2 = 21), + lists:foreach( + fun(C) -> + ClientId = proplists:get_value(clientid, emqtt:info(C)), + Topic = emqx_topic:join([<<"t">>, ClientId]), + Props = #{}, + Payload = ClientId, + Opts = [{retain, true}], + ok = emqtt:publish(C, Topic, Props, Payload, Opts), + DelayedTopic = emqx_topic:join([<<"$delayed/120">>, Topic]), + ok = emqtt:publish(C, DelayedTopic, Payload), + {ok, _, [?RC_GRANTED_QOS_0]} = emqtt:subscribe(C, Topic), + ok + end, + Node1Clients ++ Node2Clients + ), + + ?assertEqual(40, erpc:call(Node2, emqx_retainer, retained_count, [])), + ?assertEqual(NumClientsNode1, erpc:call(Node1, emqx_delayed, delayed_count, [])), + ?assertEqual(NumClientsNode2, erpc:call(Node2, emqx_delayed, delayed_count, [])), + + {ok, SRef0} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := "cluster_purge_done"}), + 15_000 + ), + ok = rpc:call(Node1, emqx_node_rebalance_purge, start, [opts(Config)]), + {ok, _} = snabbkaffe:receive_events(SRef0), + + ?assertEqual([], erpc:call(Node1, emqx_cm, all_channels, [])), + ?assertEqual([], erpc:call(Node2, emqx_cm, all_channels, [])), + ?assertEqual(0, erpc:call(Node1, emqx_retainer, retained_count, [])), + ?assertEqual(0, erpc:call(Node2, emqx_retainer, retained_count, [])), + ?assertEqual(0, erpc:call(Node1, emqx_delayed, delayed_count, [])), + ?assertEqual(0, erpc:call(Node2, emqx_delayed, delayed_count, [])), + + ok = drain_exits(Node1Clients ++ Node2Clients), + + ok. diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_status_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_status_SUITE.erl index 167c37d8c..f9c50b761 100644 --- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_status_SUITE.erl +++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_status_SUITE.erl @@ -57,7 +57,7 @@ end_per_suite(Config) -> t_cluster_status(Config) -> [CoreNode, ReplicantNode] = ?config(cluster_nodes, Config), - ok = emqx_node_rebalance_api_proto_v1:node_rebalance_evacuation_start(CoreNode, #{}), + ok = emqx_node_rebalance_api_proto_v2:node_rebalance_evacuation_start(CoreNode, #{}), ?assertMatch( #{evacuations := [_], rebalances := []}, diff --git a/changes/ee/feat-11447.en.md b/changes/ee/feat-11447.en.md new file mode 100644 index 000000000..caa808861 --- /dev/null +++ b/changes/ee/feat-11447.en.md @@ -0,0 +1 @@ +Added CLI command to wipe session and retained message data on the whole cluster. diff --git a/rel/i18n/emqx_node_rebalance_api.hocon b/rel/i18n/emqx_node_rebalance_api.hocon index 8b598134a..00346d52c 100644 --- a/rel/i18n/emqx_node_rebalance_api.hocon +++ b/rel/i18n/emqx_node_rebalance_api.hocon @@ -42,6 +42,18 @@ load_rebalance_evacuation_stop.desc: load_rebalance_evacuation_stop.label: """Stop evacuation""" +cluster_purge_start.desc: +"""Start purge process""" + +cluster_purge_start.label: +"""Start purge""" + +cluster_purge_stop.desc: +"""Stop purge process""" + +cluster_purge_stop.label: +"""Stop purge""" + param_node.desc: """Node name""" @@ -150,6 +162,12 @@ local_status_session_eviction_rate.desc: local_status_session_eviction_rate.label: """Session eviction rate""" +local_status_purge_rate.desc: +"""The rate of purging sessions, in sessions per second""" + +local_status_purge_rate.label: +"""Session purge rate""" + local_status_connection_goal.desc: """The number of connections that the node should have after the rebalance/evacuation process""" From 88f1d6eafd6078d3306a329e65a7df41d7e3b090 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 4 Sep 2023 09:17:09 -0300 Subject: [PATCH 114/119] docs: hide changelog for feature This is not going GA in 5.2.0 --- changes/ee/feat-11447.en.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 changes/ee/feat-11447.en.md diff --git a/changes/ee/feat-11447.en.md b/changes/ee/feat-11447.en.md deleted file mode 100644 index caa808861..000000000 --- a/changes/ee/feat-11447.en.md +++ /dev/null @@ -1 +0,0 @@ -Added CLI command to wipe session and retained message data on the whole cluster. From 7d26b7bc1a89e6bbd3cca526606e90ba67616763 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 4 Sep 2023 16:37:38 +0400 Subject: [PATCH 115/119] fix(router): emit clearer error when conflicting schemas in use --- apps/emqx/src/emqx_router.erl | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 2cd5ffb96..464852ceb 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -503,7 +503,19 @@ choose_schema_vsn(ConfType) -> {false, true} -> v1; {true, false} -> - v2 + v2; + {false, false} -> + ?SLOG(critical, #{ + msg => "conflicting_routing_schemas_detected_in_cluster", + configured => ConfType, + reason => + "There are records in the routing tables related to both v1 " + "and v2 storage schemas. This probably means that some nodes " + "in the cluster use v1 schema and some use v2, independently " + "of each other. The routing is likely broken. Manual intervention " + "and full cluster restart is required. This node will shut down." + }), + error(conflicting_routing_schemas_detected_in_cluster) end. is_empty(Tab) -> From 9f0f2183871366ca63ed98850428575a2987a424 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 4 Sep 2023 18:05:59 +0400 Subject: [PATCH 116/119] test(router): simplify cluster-related testcase in helper testsuite --- apps/emqx/test/emqx_cth_cluster.erl | 21 ++++++++---- apps/emqx/test/emqx_router_helper_SUITE.erl | 37 ++++----------------- 2 files changed, 22 insertions(+), 36 deletions(-) diff --git a/apps/emqx/test/emqx_cth_cluster.erl b/apps/emqx/test/emqx_cth_cluster.erl index 5e8bd4103..6d1fddb42 100644 --- a/apps/emqx/test/emqx_cth_cluster.erl +++ b/apps/emqx/test/emqx_cth_cluster.erl @@ -19,8 +19,12 @@ -export([start/2]). -export([stop/1]). +-export([start_bare_node/2]). + -export([share_load_module/2]). +-export([node_name/1]). + -define(APPS_CLUSTERING, [gen_rpc, mria, ekka]). -define(TIMEOUT_NODE_START_MS, 15000). @@ -256,9 +260,6 @@ allocate_listener_ports(Types, Spec) -> start_node_init(Spec = #{name := Node}) -> Node = start_bare_node(Node, Spec), - pong = net_adm:ping(Node), - % Preserve node spec right on the remote node - ok = set_node_opts(Node, Spec), % Make it possible to call `ct:pal` and friends (if running under rebar3) _ = share_load_module(Node, cthr), % Enable snabbkaffe trace forwarding @@ -363,7 +364,8 @@ listener_port(BasePort, wss) -> %% -start_bare_node(Name, #{driver := ct_slave}) -> +-spec start_bare_node(atom(), map()) -> node(). +start_bare_node(Name, Spec = #{driver := ct_slave}) -> {ok, Node} = ct_slave:start( node_name(Name), [ @@ -375,9 +377,15 @@ start_bare_node(Name, #{driver := ct_slave}) -> {env, []} ] ), - Node; -start_bare_node(Name, #{driver := slave}) -> + init_bare_node(Node, Spec); +start_bare_node(Name, Spec = #{driver := slave}) -> {ok, Node} = slave:start_link(host(), Name, ebin_path()), + init_bare_node(Node, Spec). + +init_bare_node(Node, Spec) -> + pong = net_adm:ping(Node), + % Preserve node spec right on the remote node + ok = set_node_opts(Node, Spec), Node. erl_flags() -> @@ -400,6 +408,7 @@ share_load_module(Node, Module) -> error end. +-spec node_name(atom()) -> node(). node_name(Name) -> case string:tokens(atom_to_list(Name), "@") of [_Name, _Host] -> diff --git a/apps/emqx/test/emqx_router_helper_SUITE.erl b/apps/emqx/test/emqx_router_helper_SUITE.erl index d65a6f666..889c8293c 100644 --- a/apps/emqx/test/emqx_router_helper_SUITE.erl +++ b/apps/emqx/test/emqx_router_helper_SUITE.erl @@ -59,33 +59,10 @@ mk_config(routing_schema_v2) -> override_env => [{boot_modules, [router]}] }. -init_per_testcase(TestCase, Config) when - TestCase =:= t_cleanup_monitor_node_down --> - ok = snabbkaffe:start_trace(), - GroupName = ?config(group_name, Config), - WorkDir = filename:join([?config(priv_dir, Config), ?MODULE, GroupName, TestCase]), - [Slave] = emqx_cth_cluster:start( - [ - {?MODULE, #{ - apps => [{emqx, mk_config(GroupName)}], - join_to => node() - }} - ], - #{work_dir => WorkDir} - ), - [{slave, Slave} | Config]; init_per_testcase(_TestCase, Config) -> ok = snabbkaffe:start_trace(), Config. -end_per_testcase(TestCase, Config) when - TestCase =:= t_cleanup_monitor_node_down --> - Slave = ?config(slave, Config), - ok = emqx_cth_cluster:stop([Slave]), - ok = snabbkaffe:stop(), - ok; end_per_testcase(_TestCase, _Config) -> ok = snabbkaffe:stop(), ok. @@ -102,8 +79,8 @@ t_mnesia(_) -> ?ROUTER_HELPER ! {membership, {mnesia, down, node()}}, ct:sleep(200). -t_cleanup_membership_mnesia_down(Config) -> - Slave = ?config(slave, Config), +t_cleanup_membership_mnesia_down(_Config) -> + Slave = emqx_cth_cluster:node_name(?FUNCTION_NAME), emqx_router:add_route(<<"a/b/c">>, Slave), emqx_router:add_route(<<"d/e/f">>, node()), ?assertMatch([_, _], emqx_router:topics()), @@ -114,8 +91,8 @@ t_cleanup_membership_mnesia_down(Config) -> ), ?assertEqual([<<"d/e/f">>], emqx_router:topics()). -t_cleanup_membership_node_down(Config) -> - Slave = ?config(slave, Config), +t_cleanup_membership_node_down(_Config) -> + Slave = emqx_cth_cluster:node_name(?FUNCTION_NAME), emqx_router:add_route(<<"a/b/c">>, Slave), emqx_router:add_route(<<"d/e/f">>, node()), ?assertMatch([_, _], emqx_router:topics()), @@ -126,13 +103,13 @@ t_cleanup_membership_node_down(Config) -> ), ?assertEqual([<<"d/e/f">>], emqx_router:topics()). -t_cleanup_monitor_node_down(Config) -> - Slave = ?config(slave, Config), +t_cleanup_monitor_node_down(_Config) -> + Slave = emqx_cth_cluster:start_bare_node(?FUNCTION_NAME, #{driver => ct_slave}), emqx_router:add_route(<<"a/b/c">>, Slave), emqx_router:add_route(<<"d/e/f">>, node()), ?assertMatch([_, _], emqx_router:topics()), ?wait_async_action( - emqx_common_test_helpers:stop_slave(Slave), + emqx_cth_cluster:stop([Slave]), #{?snk_kind := emqx_router_helper_cleanup_done, node := Slave}, 1_000 ), From 73d4248334114d90971cfc55db32a213def971f3 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 4 Sep 2023 11:22:26 -0300 Subject: [PATCH 117/119] test(cth_cluster): always allocate listener ports --- apps/emqx/test/emqx_cth_cluster.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx/test/emqx_cth_cluster.erl b/apps/emqx/test/emqx_cth_cluster.erl index 194a681d5..9ad8dc0f4 100644 --- a/apps/emqx/test/emqx_cth_cluster.erl +++ b/apps/emqx/test/emqx_cth_cluster.erl @@ -238,6 +238,8 @@ default_appspec(emqx_conf, Spec, _NodeSpecs) -> listeners => allocate_listener_ports([tcp, ssl, ws, wss], Spec) } }; +default_appspec(emqx, Spec, _NodeSpecs) -> + #{config => #{listeners => allocate_listener_ports([tcp, ssl, ws, wss], Spec)}}; default_appspec(_App, _, _) -> #{}. From 02c3d6a0a7d8d20f8f926bcfeff9dd944c90c69e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 4 Sep 2023 14:19:59 -0300 Subject: [PATCH 118/119] fix(data_import): rm duplicate import call and fix test --- apps/emqx_management/src/emqx_mgmt_data_backup.erl | 1 - apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index b83a46903..b677da2b2 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -525,7 +525,6 @@ do_import_conf(RawConf, Opts) -> Errors = lists:foldr( fun(Module, ErrorsAcc) -> - Module:import_config(RawConf), case Module:import_config(RawConf) of {ok, #{changed := Changed}} -> maybe_print_changed(Changed, Opts), diff --git a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl index f9b9ef766..d7db39657 100644 --- a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl @@ -440,8 +440,8 @@ create_test_tab(Attributes) -> apps_to_start() -> [ - {emqx_conf, "dashboard.listeners.http.bind = 0"}, {emqx, #{override_env => [{boot_modules, [broker, router]}]}}, + {emqx_conf, "dashboard.listeners.http.bind = 0"}, emqx_psk, emqx_management, emqx_dashboard, From 2cfe5ba057c433a044657dd648df893cc5702925 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 4 Sep 2023 14:37:25 -0300 Subject: [PATCH 119/119] fix(plugins_api): fix response types and test --- apps/emqx_management/src/emqx_mgmt_api_plugins.erl | 5 ++++- apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index 81c765fc4..c89ee202e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -166,7 +166,10 @@ schema("/plugins/:name/move") -> tags => ?TAGS, parameters => [hoconsc:ref(name)], 'requestBody' => move_request_body(), - responses => #{200 => <<"OK">>} + responses => #{ + 200 => <<"OK">>, + 400 => emqx_dashboard_swagger:error_codes(['MOVE_FAILED'], <<"Move failed">>) + } } }. diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl index 1cddad76d..61ed94bdc 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl @@ -297,7 +297,8 @@ update_plugin(Config, Name, Action) when is_list(Config) -> update_boot_order(Name, MoveBody, Config) -> #{host := Host, auth := Auth} = get_host_and_auth(Config), Path = emqx_mgmt_api_test_util:api_path(Host, ["plugins", Name, "move"]), - case emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, MoveBody) of + Opts = #{return_all => true}, + case emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, MoveBody, Opts) of {ok, Res} -> Resp = case emqx_utils_json:safe_decode(Res, [return_maps]) of @@ -362,8 +363,8 @@ cluster(TestCase, Config) -> app_specs(_Config) -> [ - emqx_conf, emqx, + emqx_conf, emqx_management, emqx_plugins ].