From b7c7d400e2bc792b536dfed17159a157daa4dc73 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 3 May 2023 17:45:46 +0200 Subject: [PATCH 001/144] docs: better max_overflow field label and description for MongoDB bridge The new label and description for the max_overflow field clarifies the purpose of the field, which is to manage the maximum number of additional workers created when all workers in the pool are busy, allowing more concurrent connections to the MongoDB server during workload spikes. Fixes part of: https://emqx.atlassian.net/browse/EMQX-9714 --- rel/i18n/emqx_connector_mongo.hocon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/i18n/emqx_connector_mongo.hocon b/rel/i18n/emqx_connector_mongo.hocon index bba26d736..facbab3a3 100644 --- a/rel/i18n/emqx_connector_mongo.hocon +++ b/rel/i18n/emqx_connector_mongo.hocon @@ -49,10 +49,10 @@ local_threshold.label: """Local Threshold""" max_overflow.desc: -"""Max Overflow.""" +"""The maximum number of additional workers that can be created when all workers in the pool are busy. This helps to manage temporary spikes in workload by allowing more concurrent connections to the MongoDB server.""" max_overflow.label: -"""Max Overflow""" +"""Max Overflow Workers""" min_heartbeat_period.desc: """Controls the minimum amount of time to wait between heartbeats.""" From f965fa7b9aa26b2b58b89b2445c4b308d3376339 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Mon, 1 May 2023 18:53:57 -0300 Subject: [PATCH 002/144] feat: add shutdown count Fixes https://emqx.atlassian.net/browse/EMQX-9008 Fixes https://emqx.atlassian.net/browse/EMQX-9219 --- apps/emqx/src/emqx_listeners.erl | 14 ++++++++- apps/emqx/test/emqx_client_SUITE.erl | 33 +++++++++++++++++++++- apps/emqx_management/src/emqx_mgmt_cli.erl | 7 ++++- changes/ce/feat-10568.en.md | 1 + 4 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 changes/ce/feat-10568.en.md diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index b3043effc..7ce8d4208 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -35,7 +35,8 @@ current_conns/2, max_conns/2, id_example/0, - default_max_conn/0 + default_max_conn/0, + shutdown_count/2 ]). -export([ @@ -195,6 +196,17 @@ max_conns(Type, Name, _ListenOn) when Type =:= ws; Type =:= wss -> max_conns(_, _, _) -> {error, not_support}. +shutdown_count(ID, ListenOn) -> + {ok, #{type := Type, name := Name}} = parse_listener_id(ID), + shutdown_count(Type, Name, ListenOn). + +shutdown_count(Type, Name, ListenOn) when Type == tcp; Type == ssl -> + esockd:get_shutdown_count({listener_id(Type, Name), ListenOn}); +shutdown_count(Type, _Name, _ListenOn) when Type =:= ws; Type =:= wss -> + []; +shutdown_count(_, _, _) -> + {error, not_support}. + %% @doc Start all listeners. -spec start() -> ok. start() -> diff --git a/apps/emqx/test/emqx_client_SUITE.erl b/apps/emqx/test/emqx_client_SUITE.erl index 82d4038da..ca5f53070 100644 --- a/apps/emqx/test/emqx_client_SUITE.erl +++ b/apps/emqx/test/emqx_client_SUITE.erl @@ -67,7 +67,8 @@ groups() -> %% t_keepalive, %% t_redelivery_on_reconnect, %% subscribe_failure_test, - t_dollar_topics + t_dollar_topics, + t_sub_non_utf8_topic ]}, {mqttv5, [non_parallel_tests], [t_basic_with_props_v5]}, {others, [non_parallel_tests], [ @@ -297,6 +298,36 @@ t_dollar_topics(_) -> ok = emqtt:disconnect(C), ct:pal("$ topics test succeeded"). +t_sub_non_utf8_topic(_) -> + {ok, Socket} = gen_tcp:connect({127, 0, 0, 1}, 1883, [{active, true}, binary]), + ConnPacket = emqx_frame:serialize(#mqtt_packet{ + header = #mqtt_packet_header{type = 1}, + variable = #mqtt_packet_connect{ + clientid = <<"abcdefg">> + } + }), + ok = gen_tcp:send(Socket, ConnPacket), + receive + {tcp, _, _ConnAck = <<32, 2, 0, 0>>} -> ok + after 3000 -> ct:fail({connect_ack_not_recv, process_info(self(), messages)}) + end, + SubHeader = <<130, 18, 25, 178>>, + SubTopicLen = <<0, 13>>, + %% this is not a valid utf8 topic + SubTopic = <<128, 10, 10, 12, 178, 159, 162, 47, 115, 1, 1, 1, 1>>, + SubQoS = <<1>>, + SubPacket = <>, + ok = gen_tcp:send(Socket, SubPacket), + receive + {tcp_closed, _} -> ok + after 3000 -> ct:fail({should_get_disconnected, process_info(self(), messages)}) + end, + timer:sleep(1000), + ListenerCounts = emqx_listeners:shutdown_count('tcp:default', {{0, 0, 0, 0}, 1883}), + TopicInvalidCount = proplists:get_value(topic_filter_invalid, ListenerCounts), + ?assert(is_integer(TopicInvalidCount) andalso TopicInvalidCount > 0), + ok. + %%-------------------------------------------------------------------- %% Test cases for MQTT v5 %%-------------------------------------------------------------------- diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 253d527ac..448940904 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -615,13 +615,18 @@ listeners([]) -> {error, _} -> []; MC -> [{max_conns, MC}] end, + ShutdownCount = + case emqx_listeners:shutdown_count(ID, Bind) of + {error, _} -> []; + SC -> [{shutdown_count, SC}] + end, Info = [ {listen_on, {string, emqx_listeners:format_bind(Bind)}}, {acceptors, Acceptors}, {proxy_protocol, ProxyProtocol}, {running, Running} - ] ++ CurrentConns ++ MaxConn, + ] ++ CurrentConns ++ MaxConn ++ ShutdownCount, emqx_ctl:print("~ts~n", [ID]), lists:foreach(fun indent_print/1, Info) end, diff --git a/changes/ce/feat-10568.en.md b/changes/ce/feat-10568.en.md new file mode 100644 index 000000000..dba1abdea --- /dev/null +++ b/changes/ce/feat-10568.en.md @@ -0,0 +1 @@ +Add shutdown counter information to `emqx ctl listeners` command From d03cf261363ddc6c86d499ea785497dac357b453 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Fri, 5 May 2023 08:37:44 +0800 Subject: [PATCH 003/144] chore: enable console, disable file when in dev mode --- dev | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dev b/dev index 18e4cbc93..3ac8cb5c7 100755 --- a/dev +++ b/dev @@ -43,8 +43,8 @@ OPTIONS: ENVIRONMENT VARIABLES: - PROFILE: Overriden by '-p|--profile' option, defaults to 'emqx'. - EMQX_NODE_NAME: Overriden by '-n|--name' or '-r|--remsh' option. + PROFILE: Overridden by '-p|--profile' option, defaults to 'emqx'. + EMQX_NODE_NAME: Overridden by '-n|--name' or '-r|--remsh' option. The node name of the EMQX node. Default to emqx@127.0.0.1'. EMQX_NODE_COOKIE: Erlang cookie, defaults to ~/.erlang.cookie @@ -56,6 +56,8 @@ if [ -n "${DEBUG:-}" ]; then fi export HOCON_ENV_OVERRIDE_PREFIX='EMQX_' +export EMQX_LOG__FILE__DEFAULT__ENABLE='false' +export EMQX_LOG__CONSOLE__ENABLE='true' EMQX_NODE_NAME="${EMQX_NODE_NAME:-emqx@127.0.0.1}" PROFILE="${PROFILE:-emqx}" FORCE_COMPILE=0 From 48cf089870804fe99febb0e21762d296c03c83f4 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 28 Apr 2023 21:21:04 +0200 Subject: [PATCH 004/144] fix(mqtt): drop all local messages in session deliver --- apps/emqx/src/emqx_session.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index 8b15340e9..25bee629e 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -291,16 +291,16 @@ stats(Session) -> info(?STATS_KEYS, Session). ignore_local(ClientInfo, Delivers, Subscriber, Session) -> Subs = info(subscriptions, Session), - lists:dropwhile( + lists:filter( fun({deliver, Topic, #message{from = Publisher} = Msg}) -> case maps:find(Topic, Subs) of {ok, #{nl := 1}} when Subscriber =:= Publisher -> ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, no_local]), ok = emqx_metrics:inc('delivery.dropped'), ok = emqx_metrics:inc('delivery.dropped.no_local'), - true; + false; _ -> - false + true end end, Delivers From d914d1ee1dc791df52dd18fd93b6c80936eb2155 Mon Sep 17 00:00:00 2001 From: firest Date: Sat, 6 May 2023 14:42:32 +0800 Subject: [PATCH 005/144] refactor(limiter): simplify limiter configuration --- .../emqx_limiter/src/emqx_limiter_schema.erl | 101 ++++++++++++++++-- .../emqx_limiter/src/emqx_limiter_server.erl | 9 +- .../src/emqx_limiter_server_sup.erl | 2 +- apps/emqx/src/emqx_listeners.erl | 2 +- apps/emqx/src/emqx_schema.erl | 5 +- apps/emqx/test/emqx_ratelimiter_SUITE.erl | 96 ++++++++++++++--- .../src/emqx_mgmt_api_configs.erl | 45 +++++++- rel/i18n/emqx_limiter_schema.hocon | 21 ++++ rel/i18n/emqx_schema.hocon | 21 ++++ 9 files changed, 267 insertions(+), 35 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index 40b23415c..a4f7d5b89 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -32,9 +32,13 @@ get_bucket_cfg_path/2, desc/1, types/0, + short_paths/0, calc_capacity/1, extract_with_type/2, - default_client_config/0 + default_client_config/0, + short_paths_fields/1, + get_listener_opts/1, + get_node_opts/1 ]). -define(KILOBYTE, 1024). @@ -104,15 +108,17 @@ roots() -> ]. fields(limiter) -> - [ - {Type, - ?HOCON(?R_REF(node_opts), #{ - desc => ?DESC(Type), - importance => ?IMPORTANCE_HIDDEN, - aliases => alias_of_type(Type) - })} - || Type <- types() - ] ++ + short_paths_fields(?MODULE) ++ + [ + {Type, + ?HOCON(?R_REF(node_opts), #{ + desc => ?DESC(Type), + importance => ?IMPORTANCE_HIDDEN, + required => {false, recursively}, + aliases => alias_of_type(Type) + })} + || Type <- types() + ] ++ [ %% This is an undocumented feature, and it won't be support anymore {client, @@ -203,6 +209,14 @@ fields(listener_client_fields) -> fields(Type) -> simple_bucket_field(Type). +short_paths_fields(DesModule) -> + [ + {Name, + ?HOCON(rate(), #{desc => ?DESC(DesModule, Name), required => false, example => Example})} + || {Name, Example} <- + lists:zip(short_paths(), [<<"1000/s">>, <<"1000/s">>, <<"100MB/s">>]) + ]. + desc(limiter) -> "Settings for the rate limiter."; desc(node_opts) -> @@ -236,6 +250,9 @@ get_bucket_cfg_path(Type, BucketName) -> types() -> [bytes, messages, connection, message_routing, internal]. +short_paths() -> + [max_conn_rate, messages_rate, bytes_rate]. + calc_capacity(#{rate := infinity}) -> infinity; calc_capacity(#{rate := Rate, burst := Burst}) -> @@ -266,6 +283,31 @@ default_client_config() -> failure_strategy => force }. +default_bucket_config() -> + #{ + rate => infinity, + burst => 0 + }. + +get_listener_opts(Conf) -> + Limiter = maps:get(limiter, Conf, undefined), + ShortPaths = maps:with(short_paths(), Conf), + get_listener_opts(Limiter, ShortPaths). + +get_node_opts(Type) -> + Opts = emqx:get_config([limiter, Type], default_bucket_config()), + case type_to_short_path_name(Type) of + undefined -> + Opts; + Name -> + case emqx:get_config([limiter, Name], undefined) of + undefined -> + Opts; + Rate -> + Opts#{rate := Rate} + end + end. + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- @@ -476,3 +518,42 @@ merge_client_bucket(Type, _, {ok, BucketVal}) -> #{Type => BucketVal}; merge_client_bucket(_, _, _) -> undefined. + +short_path_name_to_type(max_conn_rate) -> + connection; +short_path_name_to_type(messages_rate) -> + messages; +short_path_name_to_type(bytes_rate) -> + bytes. + +type_to_short_path_name(connection) -> + max_conn_rate; +type_to_short_path_name(messages) -> + messages_rate; +type_to_short_path_name(bytes) -> + bytes_rate; +type_to_short_path_name(_) -> + undefined. + +get_listener_opts(Limiter, ShortPaths) when map_size(ShortPaths) =:= 0 -> + Limiter; +get_listener_opts(undefined, ShortPaths) -> + convert_listener_short_paths(ShortPaths); +get_listener_opts(Limiter, ShortPaths) -> + Shorts = convert_listener_short_paths(ShortPaths), + emqx_utils_maps:deep_merge(Limiter, Shorts). + +convert_listener_short_paths(ShortPaths) -> + DefBucket = default_bucket_config(), + DefClient = default_client_config(), + Fun = fun(Name, Rate, Acc) -> + Type = short_path_name_to_type(Name), + case Name of + max_conn_rate -> + Acc#{Type => DefBucket#{rate => Rate}}; + _ -> + Client = maps:get(client, Acc, #{}), + Acc#{client => Client#{Type => DefClient#{rate => Rate}}} + end + end, + maps:fold(Fun, #{}, ShortPaths). diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 2867283d6..488f47851 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -481,7 +481,7 @@ dispatch_burst_to_buckets([], _, Alloced, Buckets) -> -spec init_tree(emqx_limiter_schema:limiter_type()) -> state(). init_tree(Type) when is_atom(Type) -> - Cfg = emqx:get_config([limiter, Type]), + Cfg = emqx_limiter_schema:get_node_opts(Type), init_tree(Type, Cfg). init_tree(Type, #{rate := Rate} = Cfg) -> @@ -625,13 +625,10 @@ find_referenced_bucket(Id, Type, #{rate := Rate} = Cfg) when Rate =/= infinity - {error, invalid_bucket} end; %% this is a node-level reference -find_referenced_bucket(Id, Type, _) -> - case emqx:get_config([limiter, Type], undefined) of +find_referenced_bucket(_Id, Type, _) -> + case emqx_limiter_schema:get_node_opts(Type) of #{rate := infinity} -> false; - undefined -> - ?SLOG(error, #{msg => "invalid limiter type", type => Type, id => Id}), - {error, invalid_bucket}; NodeCfg -> {ok, Bucket} = emqx_limiter_manager:find_root(Type), {ok, Bucket, NodeCfg} diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl index cba11ede2..be9b62d01 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl @@ -86,7 +86,7 @@ init([]) -> %% Internal functions %%--================================================================== make_child(Type) -> - Cfg = emqx:get_config([limiter, Type]), + Cfg = emqx_limiter_schema:get_node_opts(Type), make_child(Type, Cfg). make_child(Type, Cfg) -> diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 99ab52f61..1d49a0c44 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -639,7 +639,7 @@ zone(Opts) -> maps:get(zone, Opts, undefined). limiter(Opts) -> - maps:get(limiter, Opts, undefined). + emqx_limiter_schema:get_listener_opts(Opts). add_limiter_bucket(Id, #{limiter := Limiter}) -> maps:fold( diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 790cb6107..c875b9632 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2000,7 +2000,8 @@ base_listener(Bind) -> listener_fields ), #{ - desc => ?DESC(base_listener_limiter) + desc => ?DESC(base_listener_limiter), + importance => ?IMPORTANCE_HIDDEN } )}, {"enable_authn", @@ -2011,7 +2012,7 @@ base_listener(Bind) -> default => true } )} - ]. + ] ++ emqx_limiter_schema:short_paths_fields(?MODULE). desc("persistent_session_store") -> "Settings for message persistence."; diff --git a/apps/emqx/test/emqx_ratelimiter_SUITE.erl b/apps/emqx/test/emqx_ratelimiter_SUITE.erl index 67ed8e6bc..6f488eaa9 100644 --- a/apps/emqx/test/emqx_ratelimiter_SUITE.erl +++ b/apps/emqx/test/emqx_ratelimiter_SUITE.erl @@ -47,7 +47,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:load_config(emqx_limiter_schema, ?BASE_CONF), + load_conf(), emqx_common_test_helpers:start_apps([?APP]), Config. @@ -55,13 +55,15 @@ end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([?APP]). init_per_testcase(_TestCase, Config) -> + emqx_config:erase(limiter), + load_conf(), Config. end_per_testcase(_TestCase, Config) -> Config. load_conf() -> - emqx_common_test_helpers:load_config(emqx_limiter_schema, ?BASE_CONF). + ok = emqx_common_test_helpers:load_config(emqx_limiter_schema, ?BASE_CONF). init_config() -> emqx_config:init_load(emqx_limiter_schema, ?BASE_CONF). @@ -313,8 +315,8 @@ t_capacity(_) -> %% Test Cases Global Level %%-------------------------------------------------------------------- t_collaborative_alloc(_) -> - GlobalMod = fun(#{message_routing := MR} = Cfg) -> - Cfg#{message_routing := MR#{rate := ?RATE("600/1s")}} + GlobalMod = fun(Cfg) -> + Cfg#{message_routing => #{rate => ?RATE("600/1s"), burst => 0}} end, Bucket1 = fun(#{client := Cli} = Bucket) -> @@ -353,11 +355,11 @@ t_collaborative_alloc(_) -> ). t_burst(_) -> - GlobalMod = fun(#{message_routing := MR} = Cfg) -> + GlobalMod = fun(Cfg) -> Cfg#{ - message_routing := MR#{ - rate := ?RATE("200/1s"), - burst := ?RATE("400/1s") + message_routing => #{ + rate => ?RATE("200/1s"), + burst => ?RATE("400/1s") } } end, @@ -653,16 +655,16 @@ t_not_exists_instance(_) -> ), ?assertEqual( - {error, invalid_bucket}, + {ok, infinity}, emqx_limiter_server:connect(?FUNCTION_NAME, not_exists, Cfg) ), ok. t_create_instance_with_node(_) -> - GlobalMod = fun(#{message_routing := MR} = Cfg) -> + GlobalMod = fun(Cfg) -> Cfg#{ - message_routing := MR#{rate := ?RATE("200/1s")}, - messages := MR#{rate := ?RATE("200/1s")} + message_routing => #{rate => ?RATE("200/1s"), burst => 0}, + messages => #{rate => ?RATE("200/1s"), burst => 0} } end, @@ -739,6 +741,68 @@ t_esockd_htb_consume(_) -> ?assertMatch({ok, _}, C2R), ok. +%%-------------------------------------------------------------------- +%% Test Cases short paths +%%-------------------------------------------------------------------- +t_node_short_paths(_) -> + CfgStr = <<"limiter {max_conn_rate = \"1000\", messages_rate = \"100\", bytes_rate = \"10\"}">>, + ok = emqx_common_test_helpers:load_config(emqx_limiter_schema, CfgStr), + Accessor = fun emqx_limiter_schema:get_node_opts/1, + ?assertMatch(#{rate := 100.0}, Accessor(connection)), + ?assertMatch(#{rate := 10.0}, Accessor(messages)), + ?assertMatch(#{rate := 1.0}, Accessor(bytes)), + ?assertMatch(#{rate := infinity}, Accessor(message_routing)), + ?assertEqual(undefined, emqx:get_config([limiter, connection], undefined)). + +t_compatibility_for_node_short_paths(_) -> + CfgStr = + <<"limiter {max_conn_rate = \"1000\", connection.rate = \"500\", bytes.rate = \"200\"}">>, + ok = emqx_common_test_helpers:load_config(emqx_limiter_schema, CfgStr), + Accessor = fun emqx_limiter_schema:get_node_opts/1, + ?assertMatch(#{rate := 100.0}, Accessor(connection)), + ?assertMatch(#{rate := 20.0}, Accessor(bytes)). + +t_listener_short_paths(_) -> + CfgStr = << + "" + "listeners.tcp.default {max_conn_rate = \"1000\", messages_rate = \"100\", bytes_rate = \"10\"}" + "" + >>, + ok = emqx_common_test_helpers:load_config(emqx_schema, CfgStr), + ListenerOpt = emqx:get_config([listeners, tcp, default]), + ?assertMatch( + #{ + client := #{ + messages := #{rate := 10.0}, + bytes := #{rate := 1.0} + }, + connection := #{rate := 100.0} + }, + emqx_limiter_schema:get_listener_opts(ListenerOpt) + ). + +t_compatibility_for_listener_short_paths(_) -> + CfgStr = << + "" "listeners.tcp.default {max_conn_rate = \"1000\", limiter.connection.rate = \"500\"}" "" + >>, + ok = emqx_common_test_helpers:load_config(emqx_schema, CfgStr), + ListenerOpt = emqx:get_config([listeners, tcp, default]), + ?assertMatch( + #{ + connection := #{rate := 100.0} + }, + emqx_limiter_schema:get_listener_opts(ListenerOpt) + ). + +t_no_limiter_for_listener(_) -> + CfgStr = <<>>, + ok = emqx_common_test_helpers:load_config(emqx_schema, CfgStr), + ListenerOpt = emqx:get_config([listeners, tcp, default]), + ?assertEqual( + undefined, + emqx_limiter_schema:get_listener_opts(ListenerOpt) + ). + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- @@ -1043,3 +1107,11 @@ make_create_test_data_with_infinity_node(FakeInstnace) -> %% client = C bucket = B C > B {MkA(1000, 100), IsRefLimiter(FakeInstnace)} ]. + +parse_schema(ConfigString) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + hocon_tconf:check_plain( + emqx_limiter_schema, + RawConf, + #{required => false, atom_key => false} + ). diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index bc9aaf768..1d691c536 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -28,7 +28,8 @@ config_reset/3, configs/3, get_full_config/0, - global_zone_configs/3 + global_zone_configs/3, + limiter/3 ]). -define(PREFIX, "/configs/"). @@ -42,7 +43,6 @@ <<"alarm">>, <<"sys_topics">>, <<"sysmon">>, - <<"limiter">>, <<"log">>, <<"persistent_session_store">>, <<"zones">> @@ -57,7 +57,8 @@ paths() -> [ "/configs", "/configs_reset/:rootname", - "/configs/global_zone" + "/configs/global_zone", + "/configs/limiter" ] ++ lists:map(fun({Name, _Type}) -> ?PREFIX ++ binary_to_list(Name) end, config_list()). @@ -147,6 +148,28 @@ schema("/configs/global_zone") -> } } }; +schema("/configs/limiter") -> + #{ + 'operationId' => limiter, + get => #{ + tags => ?TAGS, + description => <<"Get the node-level limiter configs">>, + responses => #{ + 200 => hoconsc:mk(hoconsc:ref(emqx_limiter_schema, limiter)), + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"config not found">>) + } + }, + put => #{ + tags => ?TAGS, + description => <<"Update the node-level limiter configs">>, + 'requestBody' => hoconsc:mk(hoconsc:ref(emqx_limiter_schema, limiter)), + responses => #{ + 200 => hoconsc:mk(hoconsc:ref(emqx_limiter_schema, limiter)), + 400 => emqx_dashboard_swagger:error_codes(['UPDATE_FAILED']), + 403 => emqx_dashboard_swagger:error_codes(['UPDATE_FAILED']) + } + } + }; schema(Path) -> {RootKey, {_Root, Schema}} = find_schema(Path), #{ @@ -272,6 +295,22 @@ configs(get, Params, _Req) -> {200, Res} end. +limiter(get, _Params, _Req) -> + {200, format_limiter_config(get_raw_config(limiter))}; +limiter(put, #{body := NewConf}, _Req) -> + case emqx_conf:update([limiter], NewConf, ?OPTS) of + {ok, #{raw_config := RawConf}} -> + {200, format_limiter_config(RawConf)}; + {error, {permission_denied, Reason}} -> + {403, #{code => 'UPDATE_FAILED', message => Reason}}; + {error, Reason} -> + {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Reason)}} + end. + +format_limiter_config(RawConf) -> + Shorts = lists:map(fun erlang:atom_to_binary/1, emqx_limiter_schema:short_paths()), + maps:with(Shorts, RawConf). + conf_path_reset(Req) -> <<"/api/v5", ?PREFIX_RESET, Path/binary>> = cowboy_req:path(Req), string:lexemes(Path, "/ "). diff --git a/rel/i18n/emqx_limiter_schema.hocon b/rel/i18n/emqx_limiter_schema.hocon index c99840375..b2958ce90 100644 --- a/rel/i18n/emqx_limiter_schema.hocon +++ b/rel/i18n/emqx_limiter_schema.hocon @@ -1,5 +1,26 @@ emqx_limiter_schema { +max_conn_rate.desc: +"""Maximum connection rate.
+This is used to limit the connection rate for this node, +once the limit is reached, new connections will be deferred or refused""" +max_conn_rate.label: +"""Maximum Connection Rate""" + +messages_rate.desc: +"""Messages publish rate.
+This is used to limit the inbound message numbers for this node, +once the limit is reached, the restricted client will slow down and even be hung for a while.""" +messages_rate.label: +"""Messages Publish Rate""" + +bytes_rate.desc: +"""Data publish rate.
+This is used to limit the inbound bytes rate for this node, +once the limit is reached, the restricted client will slow down and even be hung for a while.""" +bytes_rate.label: +"""Data Publish Rate""" + bucket_cfg.desc: """Bucket Configs""" diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 76cce8e78..11da6caa3 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1033,6 +1033,27 @@ base_listener_limiter.desc: base_listener_limiter.label: """Type of the rate limit.""" +max_conn_rate.desc: +"""Maximum connection rate.
+This is used to limit the connection rate for this listener, +once the limit is reached, new connections will be deferred or refused""" +max_conn_rate.label: +"""Maximum Connection Rate""" + +messages_rate.desc: +"""Messages publish rate.
+This is used to limit the inbound message numbers for each client connected to this listener, +once the limit is reached, the restricted client will slow down and even be hung for a while.""" +messages_rate.label: +"""Messages Publish Rate""" + +bytes_rate.desc: +"""Data publish rate.
+This is used to limit the inbound bytes rate for each client connected to this listener, +once the limit is reached, the restricted client will slow down and even be hung for a while.""" +bytes_rate.label: +"""Data Publish Rate""" + persistent_session_store_backend.desc: """Database management system used to store information about persistent sessions and messages. - `builtin`: Use the embedded database (mria)""" From 197ebccf952fd352422ac99f987e84acb228e95d Mon Sep 17 00:00:00 2001 From: firest Date: Sat, 6 May 2023 16:54:44 +0800 Subject: [PATCH 006/144] chore: update changes --- changes/ce/perf-10625.en.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ce/perf-10625.en.md diff --git a/changes/ce/perf-10625.en.md b/changes/ce/perf-10625.en.md new file mode 100644 index 000000000..42e712648 --- /dev/null +++ b/changes/ce/perf-10625.en.md @@ -0,0 +1,4 @@ +Simplify limiter configuration. +- Reduce the complexity of the limiter's configuration. +e.g. now users can use `limiter.messages_rate = 1000/s` to quickly set the node-level limit for the message publish. +- Update the `configs/limiter` API to suit this refactor. From d920f415cdac28af4feb1806c3564fb46ac402ce Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 25 Apr 2023 18:07:50 +0200 Subject: [PATCH 007/144] fix(emqx_logger): fix spec for set_log_level --- apps/emqx/src/emqx_logger.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_logger.erl b/apps/emqx/src/emqx_logger.erl index 114a1af49..6087acd8a 100644 --- a/apps/emqx/src/emqx_logger.erl +++ b/apps/emqx/src/emqx_logger.erl @@ -237,7 +237,7 @@ set_log_handler_level(HandlerId, Level) -> end. %% @doc Set both the primary and all handlers level in one command --spec set_log_level(logger:handler_id()) -> ok | {error, term()}. +-spec set_log_level(logger:level()) -> ok | {error, term()}. set_log_level(Level) -> case set_primary_log_level(Level) of ok -> set_all_log_handlers_level(Level); From f2a223c1e92e13ffb7f5517fb36540b5830693cb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 8 May 2023 10:59:04 +0200 Subject: [PATCH 008/144] chore: prepare for e5.0.3 release --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index f6dcdd917..b0e987c76 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.22"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.3-rc.1"). +-define(EMQX_RELEASE_EE, "5.0.3"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). From 8803c9b326d08af4dfb781203d2e15f548776d79 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 8 May 2023 10:59:38 +0200 Subject: [PATCH 009/144] chore: make sure brod_gssapi app is included in relese package --- apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src index a4fbe5673..81b522164 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src @@ -7,7 +7,8 @@ stdlib, telemetry, wolff, - brod + brod, + brod_gssapi ]}, {env, []}, {modules, []}, From 0250718910ecb82ca437868db097fbc93781ab88 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 8 May 2023 14:04:22 +0200 Subject: [PATCH 010/144] chore: prepare for v5.0.25-rc.1 release --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 6643832a2..1377ac5c5 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.24"). +-define(EMQX_RELEASE_CE, "5.0.25-rc.1"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.3"). From eba627b365184954bd9657cf995228c9b7b4d10e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 3 May 2023 14:14:00 -0300 Subject: [PATCH 011/144] fix(buffer_worker): fix inflight count when updating inflight item --- .../src/emqx_resource_buffer_worker.erl | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 266b8df69..d9cca8f03 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -1432,16 +1432,16 @@ store_async_worker_reference(InflightTID, Ref, WorkerMRef) when ack_inflight(undefined, _Ref, _Id, _Index) -> false; ack_inflight(InflightTID, Ref, Id, Index) -> - Count = + {Count, Removed} = case ets:take(InflightTID, Ref) of [?INFLIGHT_ITEM(Ref, ?QUERY(_, _, _, _), _IsRetriable, _WorkerMRef)] -> - 1; + {1, true}; [?INFLIGHT_ITEM(Ref, [?QUERY(_, _, _, _) | _] = Batch, _IsRetriable, _WorkerMRef)] -> - length(Batch); + {length(Batch), true}; [] -> - 0 + {0, false} end, - ok = dec_inflight(InflightTID, Count), + ok = dec_inflight_remove(InflightTID, Count, Removed), IsKnownRef = (Count > 0), case IsKnownRef of true -> @@ -1469,18 +1469,28 @@ mark_inflight_items_as_retriable(Data, WorkerMRef) -> %% used to update a batch after dropping expired individual queries. update_inflight_item(InflightTID, Ref, NewBatch, NumExpired) -> _ = ets:update_element(InflightTID, Ref, {?ITEM_IDX, NewBatch}), - ok = dec_inflight(InflightTID, NumExpired). + ok = dec_inflight_update(InflightTID, NumExpired). inc_inflight(InflightTID, Count) -> _ = ets:update_counter(InflightTID, ?SIZE_REF, {2, Count}), _ = ets:update_counter(InflightTID, ?BATCH_COUNT_REF, {2, 1}), ok. -dec_inflight(_InflightTID, 0) -> +dec_inflight_remove(_InflightTID, _Count = 0, _Removed = false) -> ok; -dec_inflight(InflightTID, Count) when Count > 0 -> - _ = ets:update_counter(InflightTID, ?SIZE_REF, {2, -Count, 0, 0}), +dec_inflight_remove(InflightTID, _Count = 0, _Removed = true) -> _ = ets:update_counter(InflightTID, ?BATCH_COUNT_REF, {2, -1, 0, 0}), + ok; +dec_inflight_remove(InflightTID, Count, _Removed = true) when Count > 0 -> + %% If Count > 0, it must have been removed + _ = ets:update_counter(InflightTID, ?BATCH_COUNT_REF, {2, -1, 0, 0}), + _ = ets:update_counter(InflightTID, ?SIZE_REF, {2, -Count, 0, 0}), + ok. + +dec_inflight_update(_InflightTID, _Count = 0) -> + ok; +dec_inflight_update(InflightTID, Count) when Count > 0 -> + _ = ets:update_counter(InflightTID, ?SIZE_REF, {2, -Count, 0, 0}), ok. %%============================================================================== From eaa129d0d7a8e369d14afb9bd387941bd408dd45 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 8 May 2023 15:20:13 +0800 Subject: [PATCH 012/144] feat: rename max_message_queue_len to max_mailbox_size --- apps/emqx/src/emqx_schema.erl | 5 +++-- changes/ce/feat-10623.en.md | 1 + rel/i18n/emqx_schema.hocon | 8 ++++---- rel/i18n/zh/emqx_schema.hocon | 8 ++++---- 4 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 changes/ce/feat-10623.en.md diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 790cb6107..309a2a022 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -687,12 +687,13 @@ fields("force_shutdown") -> desc => ?DESC(force_shutdown_enable) } )}, - {"max_message_queue_len", + {"max_mailbox_size", sc( range(0, inf), #{ default => 1000, - desc => ?DESC(force_shutdown_max_message_queue_len) + aliases => [max_message_queue_len], + desc => ?DESC(force_shutdown_max_mailbox_size) } )}, {"max_heap_size", diff --git a/changes/ce/feat-10623.en.md b/changes/ce/feat-10623.en.md new file mode 100644 index 000000000..9c8ce858f --- /dev/null +++ b/changes/ce/feat-10623.en.md @@ -0,0 +1 @@ +Renamed `max_message_queue_len` to `max_mailbox_size` in the `force_shutdown` configuration. Old name is kept as an alias, so this change is backward compatible. diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 76cce8e78..f5c78cc0a 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -337,11 +337,11 @@ fields_mqtt_quic_listener_retry_memory_limit.desc: fields_mqtt_quic_listener_retry_memory_limit.label: """Retry memory limit""" -force_shutdown_max_message_queue_len.desc: -"""Maximum message queue length.""" +force_shutdown_max_mailbox_size.desc: +"""In EMQX, each online client corresponds to an individual Erlang process. The configuration value establishes a mailbox size limit for these processes. If the mailbox size surpasses this limit, the client will be automatically terminated.""" -force_shutdown_max_message_queue_len.label: -"""Maximum mailbox queue length of process.""" +force_shutdown_max_mailbox_size.label: +"""Maximum mailbox size.""" sys_heartbeat_interval.desc: """Time interval for publishing following heartbeat messages: diff --git a/rel/i18n/zh/emqx_schema.hocon b/rel/i18n/zh/emqx_schema.hocon index 1e42a4589..835372868 100644 --- a/rel/i18n/zh/emqx_schema.hocon +++ b/rel/i18n/zh/emqx_schema.hocon @@ -324,11 +324,11 @@ fields_mqtt_quic_listener_retry_memory_limit.desc: fields_mqtt_quic_listener_retry_memory_limit.label: """重试内存限制""" -force_shutdown_max_message_queue_len.desc: -"""消息队列的最大长度。""" +force_shutdown_max_mailbox_size.desc: +"""每个在线客户端在 EMQX 服务器中都是独立的一个进程。该配置可以设为单个进程的邮箱消息队列设置最大长度，当超过该上限时，客户端会被强制下线。""" -force_shutdown_max_message_queue_len.label: -"""进程邮箱消息队列的最大长度""" +force_shutdown_max_mailbox_size.label: +"""进程邮箱消息数上限""" sys_heartbeat_interval.desc: """发送心跳系统消息的间隔时间，它包括： From 04e62f6a2d0a7c8f71043ee6849b33ca7fcb6360 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 8 May 2023 16:00:55 +0800 Subject: [PATCH 013/144] test: check_oom's max_mailbox_size --- apps/emqx/src/emqx_types.erl | 2 +- apps/emqx/test/emqx_channel_SUITE.erl | 2 +- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_gateway/src/emqx_gateway_utils.erl | 2 +- apps/emqx_utils/src/emqx_utils.erl | 2 +- apps/emqx_utils/test/emqx_utils_SUITE.erl | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index 7223da245..96d75daba 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -238,7 +238,7 @@ -type stats() :: [{atom(), term()}]. -type oom_policy() :: #{ - max_message_queue_len => non_neg_integer(), + max_mailbox_size => non_neg_integer(), max_heap_size => non_neg_integer(), enable => boolean() }. diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 717614430..2b7280b32 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -31,7 +31,7 @@ force_gc_conf() -> #{bytes => 16777216, count => 16000, enable => true}. force_shutdown_conf() -> - #{enable => true, max_heap_size => 4194304, max_message_queue_len => 1000}. + #{enable => true, max_heap_size => 4194304, max_mailbox_size => 1000}. rpc_conf() -> #{ diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 0419666b4..2ffca464d 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.15"}, + {vsn, "0.1.16"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, emqx, emqx_authn, emqx_ctl]}, diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 7a0188387..ced9eff48 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -78,7 +78,7 @@ -define(DEFAULT_GC_OPTS, #{count => 1000, bytes => 1024 * 1024}). -define(DEFAULT_OOM_POLICY, #{ max_heap_size => 4194304, - max_message_queue_len => 32000 + max_mailbox_size => 32000 }). -elvis([{elvis_style, god_modules, disable}]). diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl index 5f0aa3e64..e9b2a1f9e 100644 --- a/apps/emqx_utils/src/emqx_utils.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -230,7 +230,7 @@ check_oom(Policy) -> check_oom(_Pid, #{enable := false}) -> ok; check_oom(Pid, #{ - max_message_queue_len := MaxQLen, + max_mailbox_size := MaxQLen, max_heap_size := MaxHeapSize }) -> case process_info(Pid, [message_queue_len, total_heap_size]) of diff --git a/apps/emqx_utils/test/emqx_utils_SUITE.erl b/apps/emqx_utils/test/emqx_utils_SUITE.erl index 99516b0eb..6c6bcf8d3 100644 --- a/apps/emqx_utils/test/emqx_utils_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_SUITE.erl @@ -140,7 +140,7 @@ t_index_of(_) -> t_check(_) -> Policy = #{ - max_message_queue_len => 10, + max_mailbox_size => 10, max_heap_size => 1024 * 1024 * 8, enable => true }, From cc518c81b27df570c4f64d99c6f506a38d3da010 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 8 May 2023 14:34:19 +0200 Subject: [PATCH 014/144] ci: ensure git safe dir --- .github/workflows/build_packages.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index a6bcdfaf6..63fcc2c84 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -35,6 +35,10 @@ jobs: - name: Get profile to build id: get_profile run: | + set -e + THISDIR="$(pwd)" + echo "Adding $THISDIR as safe dir for git" + git config --global --add safe.directory "${THISDIR}" tag=${{ github.ref }} if git describe --tags --match "[v|e]*" --exact; then echo "WARN: This is an exact git tag, will publish release" From 8545d3d4a7e5e750a131c902abfffba4faca206c Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 3 May 2023 12:18:30 +0200 Subject: [PATCH 015/144] test: subscribe with no_local, mixed pub from different clients --- .../emqx/test/emqx_mqtt_protocol_v5_SUITE.erl | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl index d3de74f72..fe608f600 100644 --- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -829,6 +829,42 @@ t_subscribe_no_local(Config) -> ?assertEqual(1, length(receive_messages(2))), ok = emqtt:disconnect(Client1). +t_subscribe_no_local_mixed(Config) -> + ConnFun = ?config(conn_fun, Config), + Topic = nth(1, ?TOPICS), + {ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client1), + + {ok, Client2} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:ConnFun(Client2), + + %% Given tow clients and client1 subscribe to topic with 'no local' set to true + {ok, _, [2]} = emqtt:subscribe(Client1, #{}, [{Topic, [{nl, true}, {qos, 2}]}]), + + %% When mixed publish traffic are sent from both clients (Client1 sent 6 and Client2 sent 2) + CB = {fun emqtt:sync_publish_result/3, [self(), async_res]}, + ok = emqtt:publish_async(Client1, Topic, <<"t_subscribe_no_local_mixed1">>, 0, CB), + ok = emqtt:publish_async(Client2, Topic, <<"t_subscribe_no_local_mixed2">>, 0, CB), + ok = emqtt:publish_async(Client1, Topic, <<"t_subscribe_no_local_mixed3">>, 0, CB), + ok = emqtt:publish_async(Client1, Topic, <<"t_subscribe_no_local_mixed4">>, 0, CB), + ok = emqtt:publish_async(Client1, Topic, <<"t_subscribe_no_local_mixed5">>, 0, CB), + ok = emqtt:publish_async(Client2, Topic, <<"t_subscribe_no_local_mixed6">>, 0, CB), + ok = emqtt:publish_async(Client1, Topic, <<"t_subscribe_no_local_mixed7">>, 0, CB), + ok = emqtt:publish_async(Client1, Topic, <<"t_subscribe_no_local_mixed8">>, 0, CB), + [ + receive + {async_res, Res} -> ?assertEqual(ok, Res) + end + || _ <- lists:seq(1, 8) + ], + + %% Then only two messages from clients 2 are received + PubRecvd = receive_messages(9), + ct:pal("~p", [PubRecvd]), + ?assertEqual(2, length(PubRecvd)), + ok = emqtt:disconnect(Client1), + ok = emqtt:disconnect(Client2). + t_subscribe_actions(Config) -> ConnFun = ?config(conn_fun, Config), Topic = nth(1, ?TOPICS), From fc46b81fb1de2b3ec75ae3e88bdf4fb2c523c55c Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 5 May 2023 11:00:00 +0200 Subject: [PATCH 016/144] docs: change log for #10563 --- changes/ce/fix-10563.en.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ce/fix-10563.en.md diff --git a/changes/ce/fix-10563.en.md b/changes/ce/fix-10563.en.md new file mode 100644 index 000000000..f902fb57b --- /dev/null +++ b/changes/ce/fix-10563.en.md @@ -0,0 +1,2 @@ +Corrected an issue where the no_local flag was not functioning correctly. + From b688ec6ef8d4ccbd543d6e422623693a45524b12 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 8 May 2023 14:54:08 +0200 Subject: [PATCH 017/144] ci: ensure git safe dir in build_packages --- .github/workflows/build_packages.yaml | 3 ++- .github/workflows/build_packages_cron.yaml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index a6bcdfaf6..eab9dc115 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -35,6 +35,7 @@ jobs: - name: Get profile to build id: get_profile run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" tag=${{ github.ref }} if git describe --tags --match "[v|e]*" --exact; then echo "WARN: This is an exact git tag, will publish release" @@ -229,7 +230,7 @@ jobs: ARCH: ${{ matrix.arch }} run: | set -eu - git config --global --add safe.directory "/__w/emqx/emqx" + git config --global --add safe.directory "$GITHUB_WORKSPACE" # Align path for CMake caches if [ ! "$PWD" = "/emqx" ]; then ln -s $PWD /emqx diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index 36406cd1f..7f6773f4a 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -57,6 +57,7 @@ jobs: ARCH: ${{ matrix.arch }} run: | set -eu + git config --global --add safe.directory "$GITHUB_WORKSPACE" PKGTYPES="tgz pkg" IS_ELIXIR="no" for PKGTYPE in ${PKGTYPES}; From d231e708cbdc90f0f028a8b2f47ccd7de3b0a0cf Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 24 Apr 2023 15:02:50 +0200 Subject: [PATCH 018/144] feat: add IotDB bridge --- .../docker-compose-iotdb.yaml | 31 ++ .../docker-compose-toxiproxy.yaml | 1 + .ci/docker-compose-file/toxiproxy.json | 6 + apps/emqx_bridge/src/emqx_bridge.erl | 3 +- apps/emqx_bridge/src/emqx_bridge_resource.erl | 29 ++ apps/emqx_bridge/test/emqx_bridge_testlib.erl | 350 ++++++++++++++++ apps/emqx_bridge_iotdb/.gitignore | 19 + apps/emqx_bridge_iotdb/BSL.txt | 94 +++++ apps/emqx_bridge_iotdb/README.md | 26 ++ apps/emqx_bridge_iotdb/docker-ct | 2 + .../etc/emqx_bridge_iotdb.conf | 0 .../include/emqx_bridge_iotdb.hrl | 11 + apps/emqx_bridge_iotdb/rebar.config | 14 + .../src/emqx_bridge_iotdb.app.src | 22 + .../src/emqx_bridge_iotdb.erl | 232 +++++++++++ .../src/emqx_bridge_iotdb_impl.erl | 382 ++++++++++++++++++ .../test/emqx_bridge_iotdb_impl_SUITE.erl | 229 +++++++++++ changes/ee/feat-10560.en.md | 1 + lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 17 +- mix.exs | 2 + rebar.config.erl | 2 + rel/i18n/emqx_bridge_iotdb.hocon | 77 ++++ scripts/ct/run.sh | 3 + scripts/spellcheck/dicts/emqx.txt | 2 + 24 files changed, 1551 insertions(+), 4 deletions(-) create mode 100644 .ci/docker-compose-file/docker-compose-iotdb.yaml create mode 100644 apps/emqx_bridge/test/emqx_bridge_testlib.erl create mode 100644 apps/emqx_bridge_iotdb/.gitignore create mode 100644 apps/emqx_bridge_iotdb/BSL.txt create mode 100644 apps/emqx_bridge_iotdb/README.md create mode 100644 apps/emqx_bridge_iotdb/docker-ct create mode 100644 apps/emqx_bridge_iotdb/etc/emqx_bridge_iotdb.conf create mode 100644 apps/emqx_bridge_iotdb/include/emqx_bridge_iotdb.hrl create mode 100644 apps/emqx_bridge_iotdb/rebar.config create mode 100644 apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src create mode 100644 apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl create mode 100644 apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_impl.erl create mode 100644 apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl create mode 100644 changes/ee/feat-10560.en.md create mode 100644 rel/i18n/emqx_bridge_iotdb.hocon diff --git a/.ci/docker-compose-file/docker-compose-iotdb.yaml b/.ci/docker-compose-file/docker-compose-iotdb.yaml new file mode 100644 index 000000000..2e1ea881e --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-iotdb.yaml @@ -0,0 +1,31 @@ +version: '3.9' + +services: + iotdb: + container_name: iotdb + hostname: iotdb + image: apache/iotdb:1.1.0-standalone + restart: always + environment: + - enable_rest_service=true + - cn_internal_address=iotdb + - cn_internal_port=10710 + - cn_consensus_port=10720 + - cn_target_config_node_list=iotdb:10710 + - dn_rpc_address=iotdb + - dn_internal_address=iotdb + - dn_rpc_port=6667 + - dn_mpp_data_exchange_port=10740 + - dn_schema_region_consensus_port=10750 + - dn_data_region_consensus_port=10760 + - dn_target_config_node_list=iotdb:10710 + # volumes: + # - ./data:/iotdb/data + # - ./logs:/iotdb/logs + expose: + - "18080" + # IoTDB's REST interface, uncomment for local testing + # ports: + # - "18080:18080" + networks: + - emqx_bridge diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index a1ae41e2c..d91118406 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -27,6 +27,7 @@ services: - 19042:9042 - 19142:9142 - 14242:4242 + - 28080:18080 command: - "-host=0.0.0.0" - "-config=/config/toxiproxy.json" diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index e4fbfa62a..dee3134f5 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -125,5 +125,11 @@ "listen": "0.0.0.0:1521", "upstream": "oracle:1521", "enabled": true + }, + { + "name": "iotdb", + "listen": "0.0.0.0:18080", + "upstream": "iotdb:18080", + "enabled": true } ] diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index a37b6db3c..3aade0369 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -72,7 +72,8 @@ T == cassandra; T == sqlserver; T == pulsar_producer; - T == oracle + T == oracle; + T == iotdb ). load() -> diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index da98b073e..a8dd76214 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -56,6 +56,11 @@ (TYPE) =:= <<"kafka_consumer">> orelse ?IS_BI_DIR_BRIDGE(TYPE) ). +%% [FIXME] this has no place here, it's used in parse_confs/3, which should +%% rather delegate to a behavior callback than implementing domain knowledge +%% here (reversed dependency) +-define(INSERT_TABLET_PATH, "/rest/v2/insertTablet"). + -if(?EMQX_RELEASE_EDITION == ee). bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt; bridge_to_resource_type(mqtt) -> emqx_connector_mqtt; @@ -329,6 +334,30 @@ parse_confs( max_retries => Retry } }; +parse_confs(<<"iotdb">>, Name, Conf) -> + #{ + base_url := BaseURL, + authentication := + #{ + username := Username, + password := Password + } + } = Conf, + BasicToken = base64:encode(<>), + WebhookConfig = + Conf#{ + method => <<"post">>, + url => <>, + headers => [ + {<<"Content-type">>, <<"application/json">>}, + {<<"Authorization">>, BasicToken} + ] + }, + parse_confs( + <<"webhook">>, + Name, + WebhookConfig + ); parse_confs(Type, Name, Conf) when ?IS_INGRESS_BRIDGE(Type) -> %% For some drivers that can be used as data-sources, we need to provide a %% hookpoint. The underlying driver will run `emqx_hooks:run/3` when it diff --git a/apps/emqx_bridge/test/emqx_bridge_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_testlib.erl new file mode 100644 index 000000000..47f29aa36 --- /dev/null +++ b/apps/emqx_bridge/test/emqx_bridge_testlib.erl @@ -0,0 +1,350 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_testlib). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% ct setup helpers + +init_per_suite(Config, Apps) -> + [{start_apps, Apps} | Config]. + +end_per_suite(Config) -> + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([emqx_conf]), + ok = emqx_connector_test_helpers:stop_apps(lists:reverse(?config(start_apps, Config))), + _ = application:stop(emqx_connector), + ok. + +init_per_group(TestGroup, BridgeType, Config) -> + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + application:load(emqx_bridge), + ok = emqx_common_test_helpers:start_apps([emqx_conf]), + ok = emqx_connector_test_helpers:start_apps(?config(start_apps, Config)), + {ok, _} = application:ensure_all_started(emqx_connector), + emqx_mgmt_api_test_util:init_suite(), + UniqueNum = integer_to_binary(erlang:unique_integer([positive])), + MQTTTopic = <<"mqtt/topic/", UniqueNum/binary>>, + [ + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort}, + {mqtt_topic, MQTTTopic}, + {test_group, TestGroup}, + {bridge_type, BridgeType} + | Config + ]. + +end_per_group(Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + delete_all_bridges(), + ok. + +init_per_testcase(TestCase, Config0, BridgeConfigCb) -> + ct:timetrap(timer:seconds(60)), + delete_all_bridges(), + UniqueNum = integer_to_binary(erlang:unique_integer()), + BridgeTopic = + << + (atom_to_binary(TestCase))/binary, + UniqueNum/binary + >>, + TestGroup = ?config(test_group, Config0), + Config = [{bridge_topic, BridgeTopic} | Config0], + {Name, ConfigString, BridgeConfig} = BridgeConfigCb( + TestCase, TestGroup, Config + ), + ok = snabbkaffe:start_trace(), + [ + {bridge_name, Name}, + {bridge_config_string, ConfigString}, + {bridge_config, BridgeConfig} + | Config + ]. + +end_per_testcase(_Testcase, Config) -> + case proplists:get_bool(skip_does_not_apply, Config) of + true -> + ok; + false -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + delete_all_bridges(), + %% in CI, apparently this needs more time since the + %% machines struggle with all the containers running... + emqx_common_test_helpers:call_janitor(60_000), + ok = snabbkaffe:stop(), + ok + end. + +delete_all_bridges() -> + lists:foreach( + fun(#{name := Name, type := Type}) -> + emqx_bridge:remove(Type, Name) + end, + emqx_bridge:list() + ). + +%% test helpers +parse_and_check(Config, ConfigString, Name) -> + BridgeType = ?config(bridge_type, Config), + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{BridgeType := #{Name := BridgeConfig}}} = RawConf, + BridgeConfig. + +resource_id(Config) -> + BridgeType = ?config(bridge_type, Config), + Name = ?config(bridge_name, Config), + emqx_bridge_resource:resource_id(BridgeType, Name). + +create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> + BridgeType = ?config(bridge_type, Config), + Name = ?config(bridge_name, Config), + BridgeConfig0 = ?config(bridge_config, Config), + BridgeConfig = emqx_utils_maps:deep_merge(BridgeConfig0, Overrides), + emqx_bridge:create(BridgeType, Name, BridgeConfig). + +create_bridge_api(Config) -> + create_bridge_api(Config, _Overrides = #{}). + +create_bridge_api(Config, Overrides) -> + BridgeType = ?config(bridge_type, Config), + Name = ?config(bridge_name, Config), + BridgeConfig0 = ?config(bridge_config, Config), + BridgeConfig = emqx_utils_maps:deep_merge(BridgeConfig0, Overrides), + Params = BridgeConfig#{<<"type">> => BridgeType, <<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path(["bridges"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + ct:pal("creating bridge (via http): ~p", [Params]), + Res = + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of + {ok, {Status, Headers, Body0}} -> + {ok, {Status, Headers, emqx_utils_json:decode(Body0, [return_maps])}}; + Error -> + Error + end, + ct:pal("bridge create result: ~p", [Res]), + Res. + +update_bridge_api(Config) -> + update_bridge_api(Config, _Overrides = #{}). + +update_bridge_api(Config, Overrides) -> + BridgeType = ?config(bridge_type, Config), + Name = ?config(bridge_name, Config), + BridgeConfig0 = ?config(bridge_config, Config), + BridgeConfig = emqx_utils_maps:deep_merge(BridgeConfig0, Overrides), + BridgeId = emqx_bridge_resource:bridge_id(BridgeType, Name), + Params = BridgeConfig#{<<"type">> => BridgeType, <<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path(["bridges", BridgeId]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + ct:pal("updating bridge (via http): ~p", [Params]), + Res = + case emqx_mgmt_api_test_util:request_api(put, Path, "", AuthHeader, Params, Opts) of + {ok, {_Status, _Headers, Body0}} -> {ok, emqx_utils_json:decode(Body0, [return_maps])}; + Error -> Error + end, + ct:pal("bridge update result: ~p", [Res]), + Res. + +probe_bridge_api(Config) -> + probe_bridge_api(Config, _Overrides = #{}). + +probe_bridge_api(Config, _Overrides) -> + BridgeType = ?config(bridge_type, Config), + Name = ?config(bridge_name, Config), + BridgeConfig = ?config(bridge_config, Config), + Params = BridgeConfig#{<<"type">> => BridgeType, <<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path(["bridges_probe"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + ct:pal("probing bridge (via http): ~p", [Params]), + Res = + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of + {ok, {{_, 204, _}, _Headers, _Body0} = Res0} -> {ok, Res0}; + Error -> Error + end, + ct:pal("bridge probe result: ~p", [Res]), + Res. + +create_rule_and_action_http(BridgeType, RuleTopic, Config) -> + BridgeName = ?config(bridge_name, Config), + BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName), + Params = #{ + enable => true, + sql => <<"SELECT * FROM \"", RuleTopic/binary, "\"">>, + actions => [BridgeId] + }, + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + ct:pal("rule action params: ~p", [Params]), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; + Error -> Error + end. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_sync_query(Config, MakeMessageFun, IsSuccessCheck) -> + ResourceId = resource_id(Config), + ?check_trace( + begin + ?assertMatch({ok, _}, create_bridge_api(Config)), + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + Message = {send_message, MakeMessageFun()}, + IsSuccessCheck(emqx_resource:simple_sync_query(ResourceId, Message)), + ok + end, + [] + ), + ok. + +t_async_query(Config, MakeMessageFun, IsSuccessCheck) -> + ResourceId = resource_id(Config), + ReplyFun = + fun(Pid, Result) -> + Pid ! {result, Result} + end, + ?check_trace( + begin + ?assertMatch({ok, _}, create_bridge_api(Config)), + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + Message = {send_message, MakeMessageFun()}, + emqx_resource:query(ResourceId, Message, #{async_reply_fun => {ReplyFun, [self()]}}), + ok + end, + [] + ), + receive + {result, Result} -> IsSuccessCheck(Result) + after 5_000 -> + throw(timeout) + end, + ok. + +t_create_via_http(Config) -> + ?check_trace( + begin + ?assertMatch({ok, _}, create_bridge_api(Config)), + + %% lightweight matrix testing some configs + ?assertMatch( + {ok, _}, + update_bridge_api( + Config + ) + ), + ?assertMatch( + {ok, _}, + update_bridge_api( + Config + ) + ), + ok + end, + [] + ), + ok. + +t_start_stop(Config, StopTracePoint) -> + BridgeType = ?config(bridge_type, Config), + BridgeName = ?config(bridge_name, Config), + ResourceId = resource_id(Config), + ?check_trace( + begin + ?assertMatch({ok, _}, create_bridge(Config)), + %% Since the connection process is async, we give it some time to + %% stabilize and avoid flakiness. + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + + %% Check that the bridge probe API doesn't leak atoms. + ProbeRes0 = probe_bridge_api( + Config, + #{<<"resource_opts">> => #{<<"health_check_interval">> => <<"1s">>}} + ), + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes0), + AtomsBefore = erlang:system_info(atom_count), + %% Probe again; shouldn't have created more atoms. + ProbeRes1 = probe_bridge_api( + Config, + #{<<"resource_opts">> => #{<<"health_check_interval">> => <<"1s">>}} + ), + + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes1), + AtomsAfter = erlang:system_info(atom_count), + ?assertEqual(AtomsBefore, AtomsAfter), + + %% Now stop the bridge. + ?assertMatch( + {{ok, _}, {ok, _}}, + ?wait_async_action( + emqx_bridge:disable_enable(disable, BridgeType, BridgeName), + #{?snk_kind := StopTracePoint}, + 5_000 + ) + ), + + ok + end, + fun(Trace) -> + %% one for each probe, one for real + ?assertMatch([_, _, _], ?of_kind(StopTracePoint, Trace)), + ok + end + ), + ok. + +t_on_get_status(Config) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + ResourceId = resource_id(Config), + ?assertMatch({ok, _}, create_bridge(Config)), + %% Since the connection process is async, we give it some time to + %% stabilize and avoid flakiness. + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ct:sleep(500), + ?assertEqual({ok, disconnected}, emqx_resource_manager:health_check(ResourceId)) + end), + %% Check that it recovers itself. + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + ok. diff --git a/apps/emqx_bridge_iotdb/.gitignore b/apps/emqx_bridge_iotdb/.gitignore new file mode 100644 index 000000000..e9bc1c544 --- /dev/null +++ b/apps/emqx_bridge_iotdb/.gitignore @@ -0,0 +1,19 @@ +.rebar3 + _* + .eunit + *.o + *.beam + *.plt + *.swp + *.swo + .erlang.cookie + ebin + log + erl_crash.dump + .rebar + logs + _build + .idea + *.iml + rebar3.crashdump + *~ diff --git a/apps/emqx_bridge_iotdb/BSL.txt b/apps/emqx_bridge_iotdb/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_iotdb/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_iotdb/README.md b/apps/emqx_bridge_iotdb/README.md new file mode 100644 index 000000000..48f5d74c2 --- /dev/null +++ b/apps/emqx_bridge_iotdb/README.md @@ -0,0 +1,26 @@ +# Apache IoTDB Data Integration Bridge + +This application houses the IoTDB data integration bridge for EMQX Enterprise + Edition. It provides the means to connect to IoTDB and publish messages to it. + +It implements the connection management and interaction without need for a + separate connector app, since it's not used by authentication and authorization + applications. + +# Documentation links + +For more information on Apache IoTDB, please see its [official + site](https://iotdb.apache.org/). + +# Configurations + +Please see [our official + documentation](https://www.emqx.io/docs/en/v5.0/data-integration/data-bridge-iotdb.html) + for more detailed info. + +# Contributing - [Mandatory] +Please see our [contributing.md](../../CONTRIBUTING.md). + +# License + +See [BSL](./BSL.txt). diff --git a/apps/emqx_bridge_iotdb/docker-ct b/apps/emqx_bridge_iotdb/docker-ct new file mode 100644 index 000000000..8a8973a88 --- /dev/null +++ b/apps/emqx_bridge_iotdb/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +iotdb diff --git a/apps/emqx_bridge_iotdb/etc/emqx_bridge_iotdb.conf b/apps/emqx_bridge_iotdb/etc/emqx_bridge_iotdb.conf new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_bridge_iotdb/include/emqx_bridge_iotdb.hrl b/apps/emqx_bridge_iotdb/include/emqx_bridge_iotdb.hrl new file mode 100644 index 000000000..5e6bf9ac5 --- /dev/null +++ b/apps/emqx_bridge_iotdb/include/emqx_bridge_iotdb.hrl @@ -0,0 +1,11 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_BRIDGE_IOTDB_HRL). +-define(EMQX_BRIDGE_IOTDB_HRL, true). + +-define(VSN_1_0_X, 'v1.0.x'). +-define(VSN_0_13_X, 'v0.13.x'). + +-endif. diff --git a/apps/emqx_bridge_iotdb/rebar.config b/apps/emqx_bridge_iotdb/rebar.config new file mode 100644 index 000000000..a4afd2877 --- /dev/null +++ b/apps/emqx_bridge_iotdb/rebar.config @@ -0,0 +1,14 @@ +%% -*- mode: erlang -*- + +{erl_opts, [ + debug_info +]}. + +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}}, + {emqx_bridge, {path, "../../apps/emqx_bridge"}} +]}. +{plugins, [rebar3_path_deps]}. +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src new file mode 100644 index 000000000..9c5108307 --- /dev/null +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src @@ -0,0 +1,22 @@ +%% -*- mode: erlang -*- +{application, emqx_bridge_iotdb, [ + {description, "EMQX Enterprise Apache IoTDB Bridge"}, + {vsn, "0.1.0"}, + {modules, [ + emqx_bridge_iotdb, + emqx_bridge_iotdb_impl + ]}, + {registered, []}, + {applications, [ + kernel, + stdlib, + emqx_connector + ]}, + {env, []}, + {licenses, ["Business Source License 1.1"]}, + {maintainers, ["EMQX Team "]}, + {links, [ + {"Homepage", "https://emqx.io/"}, + {"Github", "https://github.com/emqx/emqx"} + ]} +]}. diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl new file mode 100644 index 000000000..e0312bb02 --- /dev/null +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl @@ -0,0 +1,232 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_iotdb). + +-include("emqx_bridge_iotdb.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +%% hocon_schema API +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +%% emqx_ee_bridge "unofficial" API +-export([conn_bridge_examples/1]). + +%%------------------------------------------------------------------------------------------------- +%% `hocon_schema' API +%%------------------------------------------------------------------------------------------------- + +namespace() -> "bridge_iotdb". + +roots() -> []. + +fields("config") -> + basic_config() ++ request_config(); +fields("post") -> + [ + type_field(), + name_field() + ] ++ fields("config"); +fields("put") -> + fields("config"); +fields("get") -> + emqx_bridge_schema:status_fields() ++ fields("post"); +fields("creation_opts") -> + lists:filter( + fun({K, _V}) -> + not lists:member(K, unsupported_opts()) + end, + emqx_resource_schema:fields("creation_opts") + ); +fields(auth_basic) -> + [ + {username, mk(binary(), #{required => true, desc => ?DESC("config_auth_basic_username")})}, + {password, + mk(binary(), #{ + required => true, + desc => ?DESC("config_auth_basic_password"), + sensitive => true, + converter => fun emqx_schema:password_converter/2 + })} + ]. + +desc("config") -> + ?DESC("desc_config"); +desc("creation_opts") -> + ?DESC(emqx_resource_schema, "creation_opts"); +desc("post") -> + ["Configuration for IoTDB using `POST` method."]; +desc(Name) -> + lists:member(Name, struct_names()) orelse throw({missing_desc, Name}), + ?DESC(Name). + +struct_names() -> + [ + auth_basic + ]. + +basic_config() -> + [ + {enable, + mk( + boolean(), + #{ + desc => ?DESC("config_enable"), + default => true + } + )}, + {authentication, + mk( + hoconsc:union([ref(?MODULE, auth_basic)]), + #{ + default => auth_basic, desc => ?DESC("config_authentication") + } + )}, + {is_aligned, + mk( + boolean(), + #{ + desc => ?DESC("config_is_aligned"), + default => false + } + )}, + {device_id, + mk( + binary(), + #{ + desc => ?DESC("config_device_id") + } + )}, + {iotdb_version, + mk( + hoconsc:enum([?VSN_1_0_X, ?VSN_0_13_X]), + #{ + desc => ?DESC("config_iotdb_version"), + default => ?VSN_1_0_X + } + )} + ] ++ resource_creation_opts() ++ + proplists_without( + [max_retries, base_url, request], + emqx_connector_http:fields(config) + ). + +proplists_without(Keys, List) -> + [El || El = {K, _} <- List, not lists:member(K, Keys)]. + +request_config() -> + [ + {base_url, + mk( + emqx_schema:url(), + #{ + desc => ?DESC("config_base_url") + } + )}, + {max_retries, + mk( + non_neg_integer(), + #{ + default => 2, + desc => ?DESC("config_max_retries") + } + )}, + {request_timeout, + mk( + emqx_schema:duration_ms(), + #{ + default => <<"15s">>, + desc => ?DESC("config_request_timeout") + } + )} + ]. + +resource_creation_opts() -> + [ + {resource_opts, + mk( + ref(?MODULE, "creation_opts"), + #{ + required => false, + default => #{}, + desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) + } + )} + ]. + +unsupported_opts() -> + [ + batch_size, + batch_time + ]. + +%%====================================================================================== + +type_field() -> + {type, + mk( + hoconsc:enum([iotdb]), + #{ + required => true, + desc => ?DESC("desc_type") + } + )}. + +name_field() -> + {name, + mk( + binary(), + #{ + required => true, + desc => ?DESC("desc_name") + } + )}. + +%%====================================================================================== + +conn_bridge_examples(Method) -> + [ + #{ + <<"iotdb">> => + #{ + summary => <<"Apache IoTDB Bridge">>, + value => conn_bridge_example(Method, iotdb) + } + } + ]. + +conn_bridge_example(_Method, Type) -> + #{ + name => <<"My IoTDB Bridge">>, + type => Type, + enable => true, + authentication => #{ + <<"username">> => <<"root">>, + <<"password">> => <<"*****">> + }, + is_aligned => false, + device_id => <<"my_device">>, + base_url => <<"http://iotdb.local:18080/">>, + iotdb_version => ?VSN_1_0_X, + connect_timeout => <<"15s">>, + pool_type => <<"random">>, + pool_size => 8, + enable_pipelining => 100, + ssl => #{enable => false}, + resource_opts => #{ + worker_pool_size => 8, + health_check_interval => ?HEALTHCHECK_INTERVAL_RAW, + auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW, + query_mode => async, + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES + } + }. diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_impl.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_impl.erl new file mode 100644 index 000000000..2f8794560 --- /dev/null +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_impl.erl @@ -0,0 +1,382 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_iotdb_impl). + +-include("emqx_bridge_iotdb.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% `emqx_resource' API +-export([ + callback_mode/0, + on_start/2, + on_stop/2, + on_get_status/2, + on_query/3, + on_query_async/4 +]). + +-type config() :: + #{ + base_url := #{ + scheme := http | https, + host := iolist(), + port := inet:port_number(), + path := '_' + }, + connect_timeout := pos_integer(), + pool_type := random | hash, + pool_size := pos_integer(), + request := undefined | map(), + is_aligned := boolean(), + iotdb_version := binary(), + device_id := binary() | undefined, + atom() => '_' + }. + +-type state() :: + #{ + base_path := '_', + base_url := #{ + scheme := http | https, + host := iolist(), + port := inet:port_number(), + path := '_' + }, + connect_timeout := pos_integer(), + pool_type := random | hash, + pool_size := pos_integer(), + request := undefined | map(), + is_aligned := boolean(), + iotdb_version := binary(), + device_id := binary() | undefined, + atom() => '_' + }. + +-type manager_id() :: binary(). + +%%------------------------------------------------------------------------------------- +%% `emqx_resource' API +%%------------------------------------------------------------------------------------- +callback_mode() -> async_if_possible. + +-spec on_start(manager_id(), config()) -> {ok, state()} | no_return(). +on_start(InstanceId, Config) -> + %% [FIXME] The configuration passed in here is pre-processed and transformed + %% in emqx_bridge_resource:parse_confs/2. + case emqx_connector_http:on_start(InstanceId, Config) of + {ok, State} -> + ?SLOG(info, #{ + msg => "iotdb_bridge_started", + instance_id => InstanceId, + request => maps:get(request, State, <<>>) + }), + ?tp(iotdb_bridge_started, #{}), + {ok, maps:merge(Config, State)}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "failed_to_start_iotdb_bridge", + instance_id => InstanceId, + base_url => maps:get(request, Config, <<>>), + reason => Reason + }), + throw(failed_to_start_iotdb_bridge) + end. + +-spec on_stop(manager_id(), state()) -> ok | {error, term()}. +on_stop(InstanceId, State) -> + ?SLOG(info, #{ + msg => "stopping_iotdb_bridge", + connector => InstanceId + }), + Res = emqx_connector_http:on_stop(InstanceId, State), + ?tp(iotdb_bridge_stopped, #{instance_id => InstanceId}), + Res. + +-spec on_get_status(manager_id(), state()) -> + {connected, state()} | {disconnected, state(), term()}. +on_get_status(InstanceId, State) -> + emqx_connector_http:on_get_status(InstanceId, State). + +-spec on_query(manager_id(), {send_message, map()}, state()) -> + {ok, pos_integer(), [term()], term()} + | {ok, pos_integer(), [term()]} + | {error, term()}. +on_query(InstanceId, {send_message, Message}, State) -> + ?SLOG(debug, #{ + msg => "iotdb_bridge_on_query_called", + instance_id => InstanceId, + send_message => Message, + state => emqx_utils:redact(State) + }), + IoTDBPayload = make_iotdb_insert_request(Message, State), + handle_response( + emqx_connector_http:on_query( + InstanceId, {send_message, IoTDBPayload}, State + ) + ). + +-spec on_query_async(manager_id(), {send_message, map()}, {function(), [term()]}, state()) -> + {ok, pid()}. +on_query_async(InstanceId, {send_message, Message}, ReplyFunAndArgs0, State) -> + ?SLOG(debug, #{ + msg => "iotdb_bridge_on_query_async_called", + instance_id => InstanceId, + send_message => Message, + state => emqx_utils:redact(State) + }), + IoTDBPayload = make_iotdb_insert_request(Message, State), + ReplyFunAndArgs = + { + fun(Result) -> + Response = handle_response(Result), + emqx_resource:apply_reply_fun(ReplyFunAndArgs0, Response) + end, + [] + }, + emqx_connector_http:on_query_async( + InstanceId, {send_message, IoTDBPayload}, ReplyFunAndArgs, State + ). + +%%-------------------------------------------------------------------- +%% Internal Functions +%%-------------------------------------------------------------------- + +preproc_data(DataList) -> + lists:map( + fun( + #{ + measurement := Measurement, + data_type := DataType, + value := Value + } = Data + ) -> + #{ + timestamp => emqx_plugin_libs_rule:preproc_tmpl( + maps:get(<<"timestamp">>, Data, <<"now">>) + ), + measurement => emqx_plugin_libs_rule:preproc_tmpl(Measurement), + data_type => DataType, + value => emqx_plugin_libs_rule:preproc_tmpl(Value) + } + end, + DataList + ). + +proc_data(PreProcessedData, Msg) -> + NowNS = erlang:system_time(nanosecond), + Nows = #{ + now_ms => erlang:convert_time_unit(NowNS, nanosecond, millisecond), + now_us => erlang:convert_time_unit(NowNS, nanosecond, microsecond), + now_ns => NowNS + }, + lists:map( + fun( + #{ + timestamp := TimestampTkn, + measurement := Measurement, + data_type := DataType, + value := ValueTkn + } + ) -> + #{ + timestamp => iot_timestamp( + emqx_plugin_libs_rule:proc_tmpl(TimestampTkn, Msg), Nows + ), + measurement => emqx_plugin_libs_rule:proc_tmpl(Measurement, Msg), + data_type => DataType, + value => proc_value(DataType, ValueTkn, Msg) + } + end, + PreProcessedData + ). + +iot_timestamp(Timestamp, #{now_ms := NowMs}) when + Timestamp =:= <<"now">>; Timestamp =:= <<"now_ms">>; Timestamp =:= <<>> +-> + NowMs; +iot_timestamp(Timestamp, #{now_us := NowUs}) when Timestamp =:= <<"now_us">> -> + NowUs; +iot_timestamp(Timestamp, #{now_ns := NowNs}) when Timestamp =:= <<"now_ns">> -> + NowNs; +iot_timestamp(Timestamp, _) when is_binary(Timestamp) -> + binary_to_integer(Timestamp). + +proc_value(<<"TEXT">>, ValueTkn, Msg) -> + case emqx_plugin_libs_rule:proc_tmpl(ValueTkn, Msg) of + <<"undefined">> -> null; + Val -> Val + end; +proc_value(<<"BOOLEAN">>, ValueTkn, Msg) -> + convert_bool(replace_var(ValueTkn, Msg)); +proc_value(Int, ValueTkn, Msg) when Int =:= <<"INT32">>; Int =:= <<"INT64">> -> + convert_int(replace_var(ValueTkn, Msg)); +proc_value(Int, ValueTkn, Msg) when Int =:= <<"FLOAT">>; Int =:= <<"DOUBLE">> -> + convert_float(replace_var(ValueTkn, Msg)). + +replace_var(Tokens, Data) when is_list(Tokens) -> + [Val] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}), + Val; +replace_var(Val, _Data) -> + Val. + +convert_bool(B) when is_boolean(B) -> B; +convert_bool(1) -> true; +convert_bool(0) -> false; +convert_bool(<<"1">>) -> true; +convert_bool(<<"0">>) -> false; +convert_bool(<<"true">>) -> true; +convert_bool(<<"True">>) -> true; +convert_bool(<<"TRUE">>) -> true; +convert_bool(<<"false">>) -> false; +convert_bool(<<"False">>) -> false; +convert_bool(<<"FALSE">>) -> false; +convert_bool(undefined) -> null. + +convert_int(Int) when is_integer(Int) -> Int; +convert_int(Float) when is_float(Float) -> floor(Float); +convert_int(Str) when is_binary(Str) -> + try + binary_to_integer(Str) + catch + _:_ -> + convert_int(binary_to_float(Str)) + end; +convert_int(undefined) -> + null. + +convert_float(Float) when is_float(Float) -> Float; +convert_float(Int) when is_integer(Int) -> Int * 10 / 10; +convert_float(Str) when is_binary(Str) -> + try + binary_to_float(Str) + catch + _:_ -> + convert_float(binary_to_integer(Str)) + end; +convert_float(undefined) -> + null. + +make_iotdb_insert_request(Message, State) -> + IsAligned = maps:get(is_aligned, State, false), + DeviceId = device_id(Message, State), + IotDBVsn = maps:get(iotdb_version, State, ?VSN_1_0_X), + Payload = make_list(maps:get(payload, Message)), + PreProcessedData = preproc_data(Payload), + DataList = proc_data(PreProcessedData, Message), + InitAcc = #{timestamps => [], measurements => [], dtypes => [], values => []}, + Rows = replace_dtypes(aggregate_rows(DataList, InitAcc), IotDBVsn), + maps:merge(Rows, #{ + iotdb_field_key(is_aligned, IotDBVsn) => IsAligned, + iotdb_field_key(device_id, IotDBVsn) => DeviceId + }). + +replace_dtypes(Rows, IotDBVsn) -> + {Types, Map} = maps:take(dtypes, Rows), + Map#{iotdb_field_key(data_types, IotDBVsn) => Types}. + +aggregate_rows(DataList, InitAcc) -> + lists:foldr( + fun( + #{ + timestamp := Timestamp, + measurement := Measurement, + data_type := DataType, + value := Data + }, + #{ + timestamps := AccTs, + measurements := AccM, + dtypes := AccDt, + values := AccV + } = Acc + ) -> + Timestamps = [Timestamp | AccTs], + case index_of(Measurement, AccM) of + 0 -> + Acc#{ + timestamps => Timestamps, + values => [pad_value(Data, length(AccTs)) | pad_existing_values(AccV)], + measurements => [Measurement | AccM], + dtypes => [DataType | AccDt] + }; + Index -> + Acc#{ + timestamps => Timestamps, + values => insert_value(Index, Data, AccV), + measurements => AccM, + dtypes => AccDt + } + end + end, + InitAcc, + DataList + ). + +pad_value(Data, N) -> + [Data | lists:duplicate(N, null)]. + +pad_existing_values(Values) -> + [[null | Value] || Value <- Values]. + +index_of(E, List) -> + string:str(List, [E]). + +insert_value(_Index, _Data, []) -> + []; +insert_value(1, Data, [Value | Values]) -> + [[Data | Value] | insert_value(0, Data, Values)]; +insert_value(Index, Data, [Value | Values]) -> + [[null | Value] | insert_value(Index - 1, Data, Values)]. + +iotdb_field_key(is_aligned, ?VSN_1_0_X) -> + <<"is_aligned">>; +iotdb_field_key(is_aligned, ?VSN_0_13_X) -> + <<"isAligned">>; +iotdb_field_key(device_id, ?VSN_1_0_X) -> + <<"device">>; +iotdb_field_key(device_id, ?VSN_0_13_X) -> + <<"deviceId">>; +iotdb_field_key(data_types, ?VSN_1_0_X) -> + <<"data_types">>; +iotdb_field_key(data_types, ?VSN_0_13_X) -> + <<"dataTypes">>. + +make_list(List) when is_list(List) -> List; +make_list(Data) -> [Data]. + +device_id(Message, State) -> + case maps:get(device_id, State, undefined) of + undefined -> + case maps:get(payload, Message) of + #{device_id := DeviceId} -> + DeviceId; + _NotFound -> + Topic = maps:get(topic, Message), + case re:replace(Topic, "/", ".", [global, {return, binary}]) of + <<"root.", _/binary>> = Device -> Device; + Device -> <<"root.", Device/binary>> + end + end; + DeviceId -> + DeviceIdTkn = emqx_plugin_libs_rule:preproc_tmpl(DeviceId), + emqx_plugin_libs_rule:proc_tmpl(DeviceIdTkn, Message) + end. + +handle_response({ok, 200, _Headers, Body} = Resp) -> + eval_response_body(Body, Resp); +handle_response({ok, 200, Body} = Resp) -> + eval_response_body(Body, Resp); +handle_response({ok, Code, _Headers, Body}) -> + {error, #{code => Code, body => Body}}; +handle_response({ok, Code, Body}) -> + {error, #{code => Code, body => Body}}; +handle_response({error, _} = Error) -> + Error. + +eval_response_body(Body, Resp) -> + case emqx_utils_json:decode(Body) of + #{<<"code">> := 200} -> Resp; + Reason -> {error, Reason} + end. diff --git a/apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl b/apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl new file mode 100644 index 000000000..434587cf0 --- /dev/null +++ b/apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl @@ -0,0 +1,229 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_iotdb_impl_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(BRIDGE_TYPE_BIN, <<"iotdb">>). +-define(APPS, [emqx_bridge, emqx_resource, emqx_rule_engine, emqx_bridge_iotdb]). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, plain} + ]. + +groups() -> + AllTCs = emqx_common_test_helpers:all(?MODULE), + [ + {plain, AllTCs} + ]. + +init_per_suite(Config) -> + emqx_bridge_testlib:init_per_suite(Config, ?APPS). + +end_per_suite(Config) -> + emqx_bridge_testlib:end_per_suite(Config). + +init_per_group(plain = Type, Config0) -> + Host = os:getenv("IOTDB_PLAIN_HOST", "toxiproxy.emqx.net"), + Port = list_to_integer(os:getenv("IOTDB_PLAIN_PORT", "18080")), + ProxyName = "iotdb", + case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of + true -> + Config = emqx_bridge_testlib:init_per_group(Type, ?BRIDGE_TYPE_BIN, Config0), + [ + {bridge_host, Host}, + {bridge_port, Port}, + {proxy_name, ProxyName} + | Config + ]; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_iotdb); + _ -> + {skip, no_iotdb} + end + end; +init_per_group(_Group, Config) -> + Config. + +end_per_group(Group, Config) when + Group =:= plain +-> + emqx_bridge_testlib:end_per_group(Config), + ok; +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(TestCase, Config0) -> + Config = emqx_bridge_testlib:init_per_testcase(TestCase, Config0, fun bridge_config/3), + reset_service(Config), + Config. + +end_per_testcase(TestCase, Config) -> + emqx_bridge_testlib:end_per_testcase(TestCase, Config). + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +bridge_config(TestCase, _TestGroup, Config) -> + UniqueNum = integer_to_binary(erlang:unique_integer()), + Host = ?config(bridge_host, Config), + Port = ?config(bridge_port, Config), + Name = << + (atom_to_binary(TestCase))/binary, UniqueNum/binary + >>, + ServerURL = iolist_to_binary([ + "http://", + Host, + ":", + integer_to_binary(Port) + ]), + ConfigString = + io_lib:format( + "bridges.iotdb.~s {\n" + " enable = true\n" + " base_url = \"~s\"\n" + " authentication = {\n" + " username = \"root\"\n" + " password = \"root\"\n" + " }\n" + " pool_size = 1\n" + " resource_opts = {\n" + " auto_restart_interval = 5000\n" + " request_timeout = 30000\n" + " query_mode = \"async\"\n" + " worker_pool_size = 1\n" + " }\n" + "}\n", + [ + Name, + ServerURL + ] + ), + {Name, ConfigString, emqx_bridge_testlib:parse_and_check(Config, ConfigString, Name)}. + +reset_service(Config) -> + _BridgeConfig = + #{ + <<"base_url">> := BaseURL, + <<"authentication">> := #{ + <<"username">> := Username, + <<"password">> := Password + } + } = + ?config(bridge_config, Config), + ct:pal("bridge config: ~p", [_BridgeConfig]), + Path = <>, + BasicToken = base64:encode(<>), + Headers = [ + {"Content-type", "application/json"}, + {"Authorization", binary_to_list(BasicToken)} + ], + Device = iotdb_device(Config), + Body = #{sql => <<"delete from ", Device/binary, ".*">>}, + {ok, _} = emqx_mgmt_api_test_util:request_api(post, Path, "", Headers, Body, #{}). + +make_iotdb_payload(DeviceId) -> + make_iotdb_payload(DeviceId, "temp", <<"INT32">>, "36"). + +make_iotdb_payload(DeviceId, Measurement, Type, Value) -> + #{ + measurement => Measurement, + data_type => Type, + value => Value, + device_id => DeviceId, + is_aligned => false + }. + +make_message_fun(Topic, Payload) -> + fun() -> + MsgId = erlang:unique_integer([positive]), + #{ + topic => Topic, + id => MsgId, + payload => Payload, + retain => true + } + end. + +iotdb_device(Config) -> + MQTTTopic = ?config(mqtt_topic, Config), + Device = re:replace(MQTTTopic, "/", ".dev", [global, {return, binary}]), + <<"root.", Device/binary>>. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_sync_query_simple(Config) -> + DeviceId = iotdb_device(Config), + Payload = make_iotdb_payload(DeviceId, "temp", <<"INT32">>, "36"), + MakeMessageFun = make_message_fun(DeviceId, Payload), + IsSuccessCheck = + fun(Result) -> + ?assertEqual(ok, element(1, Result)) + end, + emqx_bridge_testlib:t_sync_query(Config, MakeMessageFun, IsSuccessCheck). + +t_async_query(Config) -> + DeviceId = iotdb_device(Config), + Payload = make_iotdb_payload(DeviceId, "temp", <<"INT32">>, "36"), + MakeMessageFun = make_message_fun(DeviceId, Payload), + IsSuccessCheck = + fun(Result) -> + ?assertEqual(ok, element(1, Result)) + end, + emqx_bridge_testlib:t_async_query(Config, MakeMessageFun, IsSuccessCheck). + +t_sync_query_aggregated(Config) -> + DeviceId = iotdb_device(Config), + Payload = [ + make_iotdb_payload(DeviceId, "temp", <<"INT32">>, "36"), + (make_iotdb_payload(DeviceId, "temp", <<"INT32">>, "37"))#{timestamp => <<"mow_us">>}, + (make_iotdb_payload(DeviceId, "temp", <<"INT32">>, "38"))#{timestamp => <<"mow_ns">>}, + make_iotdb_payload(DeviceId, "charged", <<"BOOLEAN">>, "1"), + make_iotdb_payload(DeviceId, "stoked", <<"BOOLEAN">>, "true"), + make_iotdb_payload(DeviceId, "enriched", <<"BOOLEAN">>, <<"TRUE">>), + make_iotdb_payload(DeviceId, "drained", <<"BOOLEAN">>, "0"), + make_iotdb_payload(DeviceId, "dazzled", <<"BOOLEAN">>, "false"), + make_iotdb_payload(DeviceId, "unplugged", <<"BOOLEAN">>, <<"FALSE">>), + make_iotdb_payload(DeviceId, "weight", <<"FLOAT">>, "87.3"), + make_iotdb_payload(DeviceId, "foo", <<"TEXT">>, <<"bar">>) + ], + MakeMessageFun = make_message_fun(DeviceId, Payload), + IsSuccessCheck = + fun(Result) -> + ?assertEqual(ok, element(1, Result)) + end, + emqx_bridge_testlib:t_sync_query(Config, MakeMessageFun, IsSuccessCheck). + +t_sync_query_fail(Config) -> + DeviceId = iotdb_device(Config), + Payload = make_iotdb_payload(DeviceId, "temp", <<"INT32">>, "Anton"), + MakeMessageFun = make_message_fun(DeviceId, Payload), + IsSuccessCheck = + fun(Result) -> + ?assertEqual(error, element(1, Result)) + end, + emqx_bridge_testlib:t_sync_query(Config, MakeMessageFun, IsSuccessCheck). + +t_create_via_http(Config) -> + emqx_bridge_testlib:t_create_via_http(Config). + +t_start_stop(Config) -> + emqx_bridge_testlib:t_start_stop(Config, iotdb_bridge_stopped). + +t_on_get_status(Config) -> + emqx_bridge_testlib:t_on_get_status(Config). diff --git a/changes/ee/feat-10560.en.md b/changes/ee/feat-10560.en.md new file mode 100644 index 000000000..c5bc59d69 --- /dev/null +++ b/changes/ee/feat-10560.en.md @@ -0,0 +1 @@ +Add enterprise data bridge for Apache IoTDB. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index c3032590e..8581f79b3 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -38,7 +38,8 @@ api_schemas(Method) -> ref(emqx_bridge_sqlserver, Method), ref(emqx_bridge_opents, Method), ref(emqx_bridge_pulsar, Method ++ "_producer"), - ref(emqx_bridge_oracle, Method) + ref(emqx_bridge_oracle, Method), + ref(emqx_bridge_iotdb, Method) ]. schema_modules() -> @@ -61,7 +62,8 @@ schema_modules() -> emqx_bridge_sqlserver, emqx_bridge_opents, emqx_bridge_pulsar, - emqx_bridge_oracle + emqx_bridge_oracle, + emqx_bridge_iotdb ]. examples(Method) -> @@ -103,7 +105,8 @@ resource_type(rocketmq) -> emqx_ee_connector_rocketmq; resource_type(sqlserver) -> emqx_bridge_sqlserver_connector; resource_type(opents) -> emqx_bridge_opents_connector; resource_type(pulsar_producer) -> emqx_bridge_pulsar_impl_producer; -resource_type(oracle) -> emqx_oracle. +resource_type(oracle) -> emqx_oracle; +resource_type(iotdb) -> emqx_bridge_iotdb_impl. fields(bridges) -> [ @@ -178,6 +181,14 @@ fields(bridges) -> desc => <<"Oracle Bridge Config">>, required => false } + )}, + {iotdb, + mk( + hoconsc:map(name, ref(emqx_bridge_iotdb, "config")), + #{ + desc => <<"Apache IoTDB Bridge Config">>, + required => false + } )} ] ++ kafka_structs() ++ pulsar_structs() ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++ diff --git a/mix.exs b/mix.exs index 0323c7896..c93ff280f 100644 --- a/mix.exs +++ b/mix.exs @@ -162,6 +162,7 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_dynamo, :emqx_bridge_hstreamdb, :emqx_bridge_influxdb, + :emqx_bridge_iotdb, :emqx_bridge_matrix, :emqx_bridge_mongodb, :emqx_bridge_mysql, @@ -372,6 +373,7 @@ defmodule EMQXUmbrella.MixProject do emqx_bridge_dynamo: :permanent, emqx_bridge_hstreamdb: :permanent, emqx_bridge_influxdb: :permanent, + emqx_bridge_iotdb: :permanent, emqx_bridge_matrix: :permanent, emqx_bridge_mongodb: :permanent, emqx_bridge_mysql: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index 524afe5bf..bb3bbbab6 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -86,6 +86,7 @@ is_community_umbrella_app("apps/emqx_bridge_clickhouse") -> false; is_community_umbrella_app("apps/emqx_bridge_dynamo") -> false; is_community_umbrella_app("apps/emqx_bridge_hstreamdb") -> false; is_community_umbrella_app("apps/emqx_bridge_influxdb") -> false; +is_community_umbrella_app("apps/emqx_bridge_iotdb") -> false; is_community_umbrella_app("apps/emqx_bridge_matrix") -> false; is_community_umbrella_app("apps/emqx_bridge_mongodb") -> false; is_community_umbrella_app("apps/emqx_bridge_mysql") -> false; @@ -463,6 +464,7 @@ relx_apps_per_edition(ee) -> emqx_bridge_dynamo, emqx_bridge_hstreamdb, emqx_bridge_influxdb, + emqx_bridge_iotdb, emqx_bridge_matrix, emqx_bridge_mongodb, emqx_bridge_mysql, diff --git a/rel/i18n/emqx_bridge_iotdb.hocon b/rel/i18n/emqx_bridge_iotdb.hocon new file mode 100644 index 000000000..d1ceecbe0 --- /dev/null +++ b/rel/i18n/emqx_bridge_iotdb.hocon @@ -0,0 +1,77 @@ +emqx_bridge_iotdb { + +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +config_authentication.desc: +"""Authentication configuration""" + +config_authentication.label: +"""Authentication""" + +auth_basic.desc: +"""Parameters for basic authentication.""" + +auth_basic.label: +"""Basic auth params""" + +config_auth_basic_username.desc: +"""The username as configured at the IoTDB REST interface""" + +config_auth_basic_username.label: + """HTTP Basic Auth Username""" + +config_auth_basic_password.desc: +"""The password as configured at the IoTDB REST interface""" + +config_auth_basic_password.label: +"""HTTP Basic Auth Password""" + +config_base_url.desc: +"""The base URL of the external IoTDB service's REST interface.""" +config_base_url.label: +"""IoTDB REST Service Base URL""" + +config_is_aligned.desc: +"""Whether to align the timeseries""" + +config_is_aligned.label: +"""Align Timeseries""" + +config_device_id.desc: +"""A fixed device name this data should be inserted for. If empty it must either be set in the rule action, the message itself, or it will be extracted from the topic.""" +config_device_id.label: +"""Device ID""" + +config_iotdb_version.desc: +"""The version of the IoTDB system to connect to.""" +config_iotdb_version.label: +"""IoTDB Version""" + +config_max_retries.desc: +"""HTTP request max retry times if failed.""" + +config_max_retries.label: +"""HTTP Request Max Retries""" + +config_request_timeout.desc: +"""HTTP request timeout.""" + +config_request_timeout.label: +"""HTTP Request Timeout""" + +desc_config.desc: +"""Configuration for Apache IoTDB bridge.""" + +desc_config.label: +"""IoTDB Bridge Configuration""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" +} diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 89082a4bd..62f616576 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -197,6 +197,9 @@ for dep in ${CT_DEPS}; do oracle) FILES+=( '.ci/docker-compose-file/docker-compose-oracle.yaml' ) ;; + iotdb) + FILES+=( '.ci/docker-compose-file/docker-compose-iotdb.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index a9afcf6ca..bdbd77b3b 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -32,6 +32,7 @@ GCM HMAC HOCON HTTPS +IoTDB JSON JWK JWKs @@ -235,6 +236,7 @@ sysmem sysmon tcp ticktime +timeseries tlog tls tlsv From b197d57a8e885e1aab0e014776c66ed4726dd9e8 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Sat, 6 May 2023 18:07:05 +0800 Subject: [PATCH 019/144] build: compatibility to make 4.4+ Prevent recursion expansion to run scripts during make. more details: https://lists.gnu.org/archive/html/info-gnu/2022-10/msg00008.html > * WARNING: Backward-incompatibility! Previously makefile variables marked as export were not exported to commands started by the $(shell ...) function. Now, all exported variables are exported to $(shell ...). If this leads to recursion during expansion, then for backward-compatibility the value from the original environment is used. To detect this change search for 'shell-export' in the .FEATURES variable. --- Makefile | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index d4a2aca34..6741317ee 100644 --- a/Makefile +++ b/Makefile @@ -4,12 +4,6 @@ SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11 export EMQX_DEFAULT_RUNNER = debian:11-slim -export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) -export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) - -export EMQX_DASHBOARD_VERSION ?= v1.2.4 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.6 - export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) @@ -19,6 +13,22 @@ else FIND=find endif +# Dashbord version +# from https://github.com/emqx/emqx-dashboard5 +export EMQX_DASHBOARD_VERSION ?= v1.2.4 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.6 + +# `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used +# In make 4.4+, for backward-compatibility the value from the original environment is used. +# so the shell script will be executed tons of times. +# https://github.com/emqx/emqx/pull/10627 +ifeq ($(strip $(OTP_VSN)),) + export OTP_VSN := $(shell $(SCRIPTS)/get-otp-vsn.sh) +endif +ifeq ($(strip $(ELIXIR_VSN)),) + export ELIXIR_VSN := $(shell $(SCRIPTS)/get-elixir-vsn.sh) +endif + PROFILE ?= emqx REL_PROFILES := emqx emqx-enterprise PKG_PROFILES := emqx-pkg emqx-enterprise-pkg From 2667f068206ba7334ede9f3ee909a888b4f34c84 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 8 May 2023 13:33:46 -0300 Subject: [PATCH 020/144] test: fix inter-suite flakiness --- lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index e2006bc6d..72bb0a7dd 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -111,12 +111,11 @@ end_per_group(_Type, _Config) -> ok. init_per_suite(Config) -> - emqx_common_test_helpers:clear_screen(), Config. end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), - ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf, emqx_rule_engine]), ok. init_per_testcase(_Testcase, Config) -> From 306a732e5e6ee9d9dfaf59ac3d6cc7ce5c14c8d2 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 8 May 2023 14:00:23 -0300 Subject: [PATCH 021/144] test: perform sanity checks when starting apps These are checks to detect inter-suite or inter-testcase flakiness early. One suite might forget one application running and stop others, and then the `application:start/2' callback is never called again for this application. One example of this was that: i) `emqx_rule_engine` was left running by one suite; ii) `emqx` app was stopped, taking `emqx_config_handler` down with it and losing the rule engine handler; iii) another suite that uses rule engine "started" it (a no-op) and then the config handler was never installed again. --- apps/emqx/test/emqx_common_test_helpers.erl | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 61373f638..c8ef40925 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -251,6 +251,7 @@ start_app(App, SpecAppConfig, Opts) -> {ok, _} -> ok = ensure_dashboard_listeners_started(App), ok = wait_for_app_processes(App), + ok = perform_sanity_checks(App), ok; {error, Reason} -> error({failed_to_start_app, App, Reason}) @@ -264,6 +265,27 @@ wait_for_app_processes(emqx_conf) -> wait_for_app_processes(_) -> ok. +%% These are checks to detect inter-suite or inter-testcase flakiness +%% early. For example, one suite might forget one application running +%% and stop others, and then the `application:start/2' callback is +%% never called again for this application. +perform_sanity_checks(emqx_rule_engine) -> + ensure_config_handler(emqx_rule_engine, [rule_engine, rules]), + ok; +perform_sanity_checks(emqx_bridge) -> + ensure_config_handler(emqx_bridge, [bridges]), + ok; +perform_sanity_checks(_App) -> + ok. + +ensure_config_handler(Module, ConfigPath) -> + #{handlers := Handlers} = sys:get_state(emqx_config_handler), + case emqx_utils_maps:deep_get(ConfigPath, Handlers, not_found) of + #{{mod} := Module} -> ok; + _NotFound -> error({config_handler_missing, ConfigPath, Module}) + end, + ok. + app_conf_file(emqx_conf) -> "emqx.conf.all"; app_conf_file(App) -> atom_to_list(App) ++ ".conf". From 77176787ca11e764f4af26d85e9198c43087e5b5 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Wed, 3 May 2023 18:30:16 -0300 Subject: [PATCH 022/144] feat: add log level configuration to ssl communication Fixes https://emqx.atlassian.net/browse/EMQX-9781 --- apps/emqx/src/emqx_schema.erl | 11 +++++++++++ changes/ce/feat-10584.en.md | 1 + rel/i18n/emqx_schema.hocon | 5 +++++ 3 files changed, 17 insertions(+) create mode 100644 changes/ce/feat-10584.en.md diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 188c22d78..76eeeb7c8 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2285,6 +2285,17 @@ common_ssl_opts_schema(Defaults) -> desc => ?DESC(common_ssl_opts_schema_secure_renegotiate) } )}, + {"log_level", + sc( + hoconsc:enum([ + emergency, alert, critical, error, warning, notice, info, debug, none, all + ]), + #{ + default => notice, + desc => ?DESC(common_ssl_opts_schema_log_level), + importance => ?IMPORTANCE_LOW + } + )}, {"hibernate_after", sc( diff --git a/changes/ce/feat-10584.en.md b/changes/ce/feat-10584.en.md new file mode 100644 index 000000000..abb514cbb --- /dev/null +++ b/changes/ce/feat-10584.en.md @@ -0,0 +1 @@ +Add log level configuration to SSL communication diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 76cce8e78..ad2fcbaeb 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1300,6 +1300,11 @@ you drop support for the insecure renegotiation, prone to MitM attacks.""" common_ssl_opts_schema_secure_renegotiate.label: """SSL renegotiate""" +common_ssl_opts_schema_log_level.desc: +"""Log level for SSL communication. Default is 'notice'. Set to 'debug' to inspect TLS handshake messages.""" +common_ssl_opts_schema_log_level.label: +"""SSL log level""" + sysmon_vm_busy_port.desc: """When a port (e.g. TCP socket) is overloaded, there will be a busy_port warning log, and an MQTT message is published to the system topic $SYS/sysmon/busy_port.""" From 4456a32465079f98dcbc71fd24de8d410234834c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 8 May 2023 22:08:51 +0200 Subject: [PATCH 023/144] chore: bump ee version to e5.0.4-alpha.1 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 1377ac5c5..2bb5877f1 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.25-rc.1"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.3"). +-define(EMQX_RELEASE_EE, "5.0.4-alpha.1"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). From 2df941015be7e4cedc3ec60ea521b6cf24ffac08 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 9 May 2023 00:15:04 +0200 Subject: [PATCH 024/144] chore: bump chart versions --- deploy/charts/emqx-enterprise/Chart.yaml | 4 ++-- deploy/charts/emqx/Chart.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index 85bd20f6e..7bc90eff7 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.3 +version: 5.0.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.3 +appVersion: 5.0.4 diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 9c23f7c15..ee2ae4be2 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.24 +version: 5.0.25 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.24 +appVersion: 5.0.25 From 7b000157d033f8cde85ae163d75697af3c658fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Fri, 5 May 2023 00:35:27 +0800 Subject: [PATCH 025/144] feat: refactor log configuration --- apps/emqx/src/config/emqx_config_logger.erl | 40 ++--- apps/emqx_conf/src/emqx_conf_schema.erl | 144 ++++++++---------- .../emqx_conf/test/emqx_conf_schema_tests.erl | 97 ++++++++++++ bin/emqx | 12 +- 4 files changed, 182 insertions(+), 111 deletions(-) diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl index b0fc1ca67..901056c0f 100644 --- a/apps/emqx/src/config/emqx_config_logger.erl +++ b/apps/emqx/src/config/emqx_config_logger.erl @@ -112,8 +112,8 @@ update_log_handler({Action, {handler, Id, Mod, Conf}}) -> end, ok. -id_for_log(console) -> "log.console_handler"; -id_for_log(Other) -> "log.file_handlers." ++ atom_to_list(Other). +id_for_log(console) -> "log.console"; +id_for_log(Other) -> "log.file." ++ atom_to_list(Other). atom(Id) when is_binary(Id) -> binary_to_atom(Id, utf8); atom(Id) when is_atom(Id) -> Id. @@ -126,12 +126,12 @@ tr_handlers(Conf) -> %% For the default logger that outputs to console tr_console_handler(Conf) -> - case conf_get("log.console_handler.enable", Conf) of + case conf_get("log.console.enable", Conf) of true -> - ConsoleConf = conf_get("log.console_handler", Conf), + ConsoleConf = conf_get("log.console", Conf), [ {handler, console, logger_std_h, #{ - level => conf_get("log.console_handler.level", Conf), + level => conf_get("log.console.level", Conf), config => (log_handler_conf(ConsoleConf))#{type => standard_io}, formatter => log_formatter(ConsoleConf), filters => log_filter(ConsoleConf) @@ -150,14 +150,10 @@ tr_file_handler({HandlerName, SubConf}) -> {handler, atom(HandlerName), logger_disk_log_h, #{ level => conf_get("level", SubConf), config => (log_handler_conf(SubConf))#{ - type => - case conf_get("rotation.enable", SubConf) of - true -> wrap; - _ -> halt - end, - file => conf_get("file", SubConf), - max_no_files => conf_get("rotation.count", SubConf), - max_no_bytes => conf_get("max_size", SubConf) + type => wrap, + file => conf_get("sink_to", SubConf), + max_no_files => conf_get("rotation_count", SubConf), + max_no_bytes => conf_get("rotation_size", SubConf) }, formatter => log_formatter(SubConf), filters => log_filter(SubConf), @@ -165,15 +161,7 @@ tr_file_handler({HandlerName, SubConf}) -> }}. logger_file_handlers(Conf) -> - Handlers = maps:to_list(conf_get("log.file_handlers", Conf, #{})), - lists:filter( - fun({_Name, Opts}) -> - B = conf_get("enable", Opts), - true = is_boolean(B), - B - end, - Handlers - ). + maps:to_list(conf_get("log.file", Conf, #{})). conf_get(Key, Conf) -> emqx_schema:conf_get(Key, Conf). conf_get(Key, Conf, Default) -> emqx_schema:conf_get(Key, Conf, Default). @@ -237,12 +225,8 @@ log_filter(Conf) -> end. tr_level(Conf) -> - ConsoleLevel = conf_get("log.console_handler.level", Conf, undefined), - FileLevels = [ - conf_get("level", SubConf) - || {_, SubConf} <- - logger_file_handlers(Conf) - ], + ConsoleLevel = conf_get("log.console.level", Conf, undefined), + FileLevels = [conf_get("level", SubConf) || {_, SubConf} <- logger_file_handlers(Conf)], case FileLevels ++ [ConsoleLevel || ConsoleLevel =/= undefined] of %% warning is the default level we should use [] -> warning; diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 94cbfb221..09f51a3b7 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -687,11 +687,12 @@ fields("rpc") -> desc => ?DESC(rpc_mode) } )}, - {"driver", + {"protocol", sc( hoconsc:enum([tcp, ssl]), #{ mapping => "gen_rpc.driver", + aliases => [driver], default => tcp, desc => ?DESC(rpc_driver) } @@ -866,19 +867,22 @@ fields("rpc") -> ]; fields("log") -> [ - {"console_handler", + {"console", + sc(?R_REF("console_handler"), #{ + aliases => [console_handler], + importance => ?IMPORTANCE_HIGH + })}, + {"file", sc( - ?R_REF("console_handler"), - #{importance => ?IMPORTANCE_HIGH} - )}, - {"file_handlers", - sc( - map(name, ?R_REF("log_file_handler")), + ?UNION([ + ?R_REF("log_file_handler"), + ?MAP(handler_name, ?R_REF("log_file_handler")) + ]), #{ desc => ?DESC("log_file_handlers"), - %% because file_handlers is a map - %% so there has to be a default value in order to populate the raw configs - default => #{<<"default">> => #{<<"level">> => <<"warning">>}}, + converter => fun ensure_file_handlers/2, + default => #{<<"level">> => <<"warning">>}, + aliases => [file_handlers], importance => ?IMPORTANCE_HIGH } )} @@ -887,50 +891,39 @@ fields("console_handler") -> log_handler_common_confs(console); fields("log_file_handler") -> [ - {"file", + {"sink_to", sc( file(), #{ desc => ?DESC("log_file_handler_file"), default => <<"${EMQX_LOG_DIR}/emqx.log">>, converter => fun emqx_schema:naive_env_interpolation/1, - validator => fun validate_file_location/1 + validator => fun validate_file_location/1, + aliases => [file], + importance => ?IMPORTANCE_HIGH } )}, - {"rotation", + {"rotation_count", sc( - ?R_REF("log_rotation"), - #{} + range(1, 128), + #{ + aliases => [rotation], + default => 10, + desc => ?DESC("log_rotation_count"), + importance => ?IMPORTANCE_MEDIUM + } )}, - {"max_size", + {"rotation_size", sc( hoconsc:union([infinity, emqx_schema:bytesize()]), #{ default => <<"50MB">>, desc => ?DESC("log_file_handler_max_size"), + aliases => [max_size], importance => ?IMPORTANCE_MEDIUM } )} ] ++ log_handler_common_confs(file); -fields("log_rotation") -> - [ - {"enable", - sc( - boolean(), - #{ - default => true, - desc => ?DESC("log_rotation_enable") - } - )}, - {"count", - sc( - range(1, 2048), - #{ - default => 10, - desc => ?DESC("log_rotation_count") - } - )} - ]; fields("log_overload_kill") -> [ {"enable", @@ -1038,8 +1031,8 @@ translation("ekka") -> [{"cluster_discovery", fun tr_cluster_discovery/1}]; translation("kernel") -> [ - {"logger_level", fun tr_logger_level/1}, - {"logger", fun tr_logger_handlers/1}, + {"logger_level", fun emqx_config_logger:tr_level/1}, + {"logger", fun emqx_config_logger:tr_handlers/1}, {"error_logger", fun(_) -> silent end} ]; translation("emqx") -> @@ -1113,24 +1106,9 @@ tr_cluster_discovery(Conf) -> Strategy = conf_get("cluster.discovery_strategy", Conf), {Strategy, filter(cluster_options(Strategy, Conf))}. --spec tr_logger_level(hocon:config()) -> logger:level(). -tr_logger_level(Conf) -> - emqx_config_logger:tr_level(Conf). - -tr_logger_handlers(Conf) -> - emqx_config_logger:tr_handlers(Conf). - log_handler_common_confs(Handler) -> - lists:map( - fun - ({_Name, #{importance := _}} = F) -> F; - ({Name, Sc}) -> {Name, Sc#{importance => ?IMPORTANCE_LOW}} - end, - do_log_handler_common_confs(Handler) - ). -do_log_handler_common_confs(Handler) -> %% we rarely support dynamic defaults like this - %% for this one, we have build-time defualut the same as runtime default + %% for this one, we have build-time default the same as runtime default %% so it's less tricky EnableValues = case Handler of @@ -1140,21 +1118,31 @@ do_log_handler_common_confs(Handler) -> EnvValue = os:getenv("EMQX_DEFAULT_LOG_HANDLER"), Enable = lists:member(EnvValue, EnableValues), [ + {"level", + sc( + log_level(), + #{ + default => warning, + desc => ?DESC("common_handler_level"), + importance => ?IMPORTANCE_HIGH + } + )}, {"enable", sc( boolean(), #{ default => Enable, desc => ?DESC("common_handler_enable"), - importance => ?IMPORTANCE_LOW + importance => ?IMPORTANCE_MEDIUM } )}, - {"level", + {"formatter", sc( - log_level(), + hoconsc:enum([text, json]), #{ - default => warning, - desc => ?DESC("common_handler_level") + default => text, + desc => ?DESC("common_handler_formatter"), + importance => ?IMPORTANCE_MEDIUM } )}, {"time_offset", @@ -1173,16 +1161,7 @@ do_log_handler_common_confs(Handler) -> #{ default => unlimited, desc => ?DESC("common_handler_chars_limit"), - importance => ?IMPORTANCE_LOW - } - )}, - {"formatter", - sc( - hoconsc:enum([text, json]), - #{ - default => text, - desc => ?DESC("common_handler_formatter"), - importance => ?IMPORTANCE_MEDIUM + importance => ?IMPORTANCE_HIDDEN } )}, {"single_line", @@ -1191,7 +1170,7 @@ do_log_handler_common_confs(Handler) -> #{ default => true, desc => ?DESC("common_handler_single_line"), - importance => ?IMPORTANCE_LOW + importance => ?IMPORTANCE_HIDDEN } )}, {"sync_mode_qlen", @@ -1199,7 +1178,8 @@ do_log_handler_common_confs(Handler) -> non_neg_integer(), #{ default => 100, - desc => ?DESC("common_handler_sync_mode_qlen") + desc => ?DESC("common_handler_sync_mode_qlen"), + importance => ?IMPORTANCE_HIDDEN } )}, {"drop_mode_qlen", @@ -1207,7 +1187,8 @@ do_log_handler_common_confs(Handler) -> pos_integer(), #{ default => 3000, - desc => ?DESC("common_handler_drop_mode_qlen") + desc => ?DESC("common_handler_drop_mode_qlen"), + importance => ?IMPORTANCE_HIDDEN } )}, {"flush_qlen", @@ -1215,17 +1196,19 @@ do_log_handler_common_confs(Handler) -> pos_integer(), #{ default => 8000, - desc => ?DESC("common_handler_flush_qlen") + desc => ?DESC("common_handler_flush_qlen"), + importance => ?IMPORTANCE_HIDDEN } )}, - {"overload_kill", sc(?R_REF("log_overload_kill"), #{})}, - {"burst_limit", sc(?R_REF("log_burst_limit"), #{})}, + {"overload_kill", sc(?R_REF("log_overload_kill"), #{importance => ?IMPORTANCE_HIDDEN})}, + {"burst_limit", sc(?R_REF("log_burst_limit"), #{importance => ?IMPORTANCE_HIDDEN})}, {"supervisor_reports", sc( hoconsc:enum([error, progress]), #{ default => error, - desc => ?DESC("common_handler_supervisor_reports") + desc => ?DESC("common_handler_supervisor_reports"), + importance => ?IMPORTANCE_HIDDEN } )}, {"max_depth", @@ -1233,7 +1216,8 @@ do_log_handler_common_confs(Handler) -> hoconsc:union([unlimited, non_neg_integer()]), #{ default => 100, - desc => ?DESC("common_handler_max_depth") + desc => ?DESC("common_handler_max_depth"), + importance => ?IMPORTANCE_HIDDEN } )} ]. @@ -1355,3 +1339,9 @@ validator_string_re(Val, RE, Error) -> node_array() -> hoconsc:union([emqx_schema:comma_separated_atoms(), hoconsc:array(atom())]). + +ensure_file_handlers(Conf, _Opts) -> + FileFields = lists:map(fun({F, _}) -> list_to_binary(F) end, fields("log_file_handler")), + HandlersWithoutName = maps:with(FileFields, Conf), + HandlersWithName = maps:without(FileFields, Conf), + emqx_utils_maps:deep_merge(#{<<"default">> => HandlersWithoutName}, HandlersWithName). diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 667d1766f..0e88da318 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -47,6 +47,103 @@ array_nodes_test() -> ), ok. +%% erlfmt-ignore +-define(OUTDATED_LOG_CONF, + """ +console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = 1000 + } + chars_limit = unlimited + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = text + level = warning + max_depth = 100 + overload_kill { + enable = true + mem_size = 31457280 + qlen = 20000 + restart_after = 5000 + } + single_line = true + supervisor_reports = error + sync_mode_qlen = 100 + time_offset = system +} +file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = 1000 + } + chars_limit = unlimited + drop_mode_qlen = 3000 + enable = false + file = \"log/emqx.log\" + flush_qlen = 8000 + formatter = text + level = warning + max_depth = 100 + max_size = 52428800 + overload_kill { + enable = true + mem_size = 31457280 + qlen = 20000 + restart_after = 5000 + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = error + sync_mode_qlen = 100 + time_offset = \"+01:00\" + } +} + """ +). + +outdated_log_test() -> + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + Conf0 = <>, + {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), + ConfList = hocon_tconf:generate(emqx_conf_schema, ConfMap0), + ct:pal("fff:~p", [ConfList]), + Log = proplists:get_value(log, ConfList), + Console = proplists:get_value(console, Log), + File = proplists:get_value(file, Log), + ?assertEqual(1, Console, {Console, File}), + ok. + +-define(NEW_LOG_CONF, + "" + "\n" + "console {\n" + " enable = true\n" + " formatter = text\n" + " level = warning\n" + " time_offset = system\n" + "}\n" + "file {\n" + " enable = true\n" + " file = \"log/emqx.log\"\n" + " formatter = text\n" + " level = warning\n" + " rotation {count = 10, enable = true}\n" + " time_offset = \"+01:00\"\n" + " }\n" + "file_handlers.default {\n" + " enable = false,\n" + " file = \"log/file_handlres_emqx.log\"\n" + " }\n" + "}\n" + " " + "" +). + %% erlfmt-ignore -define(BASE_AUTHN_ARRAY, """ diff --git a/bin/emqx b/bin/emqx index 3b7212c99..07684af8e 100755 --- a/bin/emqx +++ b/bin/emqx @@ -875,16 +875,16 @@ tr_log_to_env() { unset EMQX_LOG__TO case "${log_to}" in console) - export EMQX_LOG__CONSOLE_HANDLER__ENABLE='true' - export EMQX_LOG__FILE_HANDLERS__DEFAULT__ENABLE='false' + export EMQX_LOG__CONSOLE__ENABLE='true' + export EMQX_LOG__FILE__ENABLE='false' ;; file) - export EMQX_LOG__CONSOLE_HANDLER__ENABLE='false' - export EMQX_LOG__FILE_HANDLERS__DEFAULT__ENABLE='true' + export EMQX_LOG__CONSOLE__ENABLE='false' + export EMQX_LOG__FILE__ENABLE='true' ;; both) - export EMQX_LOG__CONSOLE_HANDLER__ENABLE='true' - export EMQX_LOG__FILE_HANDLERS__DEFAULT__ENABLE='true' + export EMQX_LOG__CONSOLE__ENABLE='true' + export EMQX_LOG__FILE__ENABLE='true' ;; default) # want to use config file defaults, do nothing From d31a44f96b457dc1b597119c26caa7dadc23913f Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 5 May 2023 09:53:54 +0800 Subject: [PATCH 026/144] feat: convert rotation from map to integer --- apps/emqx_conf/src/emqx_conf_schema.erl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 09f51a3b7..682e8f59d 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -909,6 +909,7 @@ fields("log_file_handler") -> #{ aliases => [rotation], default => 10, + converter => fun convert_rotation/2, desc => ?DESC("log_rotation_count"), importance => ?IMPORTANCE_MEDIUM } @@ -1345,3 +1346,7 @@ ensure_file_handlers(Conf, _Opts) -> HandlersWithoutName = maps:with(FileFields, Conf), HandlersWithName = maps:without(FileFields, Conf), emqx_utils_maps:deep_merge(#{<<"default">> => HandlersWithoutName}, HandlersWithName). + +convert_rotation(undefined, _Opts) -> undefined; +convert_rotation(#{} = Rotation, _Opts) -> maps:get(count, Rotation, 10); +convert_rotation(Count, _Opts) when is_integer(Count) -> Count. From bf87aebbba132b13995a1dfea4d27c4f025684e0 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 5 May 2023 10:12:28 +0800 Subject: [PATCH 027/144] chore: upgrade hocon to 0.39.5 --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 18119607e..8e51a4079 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.1"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.4"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.5"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/mix.exs b/mix.exs index 9383fa085..cdacb2986 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.39.4", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.39.5", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 03b03345d..738b3ffb0 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.4"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.5"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From 14952658d5f910ecf2b271149954e0f503c30452 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 5 May 2023 16:29:43 +0800 Subject: [PATCH 028/144] chore: convert rotation struct to roation_count --- .../emqx_bridge_compatible_config_tests.erl | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 10 +- .../emqx_conf/test/emqx_conf_schema_tests.erl | 163 ++++++++++++------ 3 files changed, 123 insertions(+), 52 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl b/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl index acafb84ca..08bee15f3 100644 --- a/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl @@ -129,7 +129,7 @@ assert_upgraded1(Map) -> ?assert(maps:is_key(<<"ssl">>, Map)). check(Conf) when is_map(Conf) -> - hocon_tconf:check_plain(emqx_bridge_schema, Conf). + hocon_tconf:check_plain(emqx_bridge_schema, Conf, #{required => false}). %% erlfmt-ignore %% this is config generated from v5.0.11 diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 682e8f59d..0e41d598c 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1342,11 +1342,17 @@ node_array() -> hoconsc:union([emqx_schema:comma_separated_atoms(), hoconsc:array(atom())]). ensure_file_handlers(Conf, _Opts) -> - FileFields = lists:map(fun({F, _}) -> list_to_binary(F) end, fields("log_file_handler")), + FileFields = lists:flatmap( + fun({F, Schema}) -> + Alias = [atom_to_binary(A) || A <- maps:get(aliases, Schema, [])], + [list_to_binary(F) | Alias] + end, + fields("log_file_handler") + ), HandlersWithoutName = maps:with(FileFields, Conf), HandlersWithName = maps:without(FileFields, Conf), emqx_utils_maps:deep_merge(#{<<"default">> => HandlersWithoutName}, HandlersWithName). convert_rotation(undefined, _Opts) -> undefined; -convert_rotation(#{} = Rotation, _Opts) -> maps:get(count, Rotation, 10); +convert_rotation(#{} = Rotation, _Opts) -> maps:get(<<"count">>, Rotation, 10); convert_rotation(Count, _Opts) when is_integer(Count) -> Count. diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 0e88da318..b0eea9a07 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -50,7 +50,7 @@ array_nodes_test() -> %% erlfmt-ignore -define(OUTDATED_LOG_CONF, """ -console_handler { +log.console_handler { burst_limit { enable = true max_count = 10000 @@ -72,9 +72,9 @@ console_handler { single_line = true supervisor_reports = error sync_mode_qlen = 100 - time_offset = system + time_offset = \"+02:00\" } -file_handlers { +log.file_handlers { default { burst_limit { enable = true @@ -84,19 +84,19 @@ file_handlers { chars_limit = unlimited drop_mode_qlen = 3000 enable = false - file = \"log/emqx.log\" + file = \"log/my-emqx.log\" flush_qlen = 8000 formatter = text - level = warning + level = debug max_depth = 100 - max_size = 52428800 + max_size = \"1024MB\" overload_kill { enable = true mem_size = 31457280 qlen = 20000 restart_after = 5000 } - rotation {count = 10, enable = true} + rotation {count = 20, enable = true} single_line = true supervisor_reports = error sync_mode_qlen = 100 @@ -105,45 +105,98 @@ file_handlers { } """ ). +-define(FORMATTER(TimeOffset), + {emqx_logger_textfmt, #{ + chars_limit => unlimited, + depth => 100, + single_line => true, + template => [time, " [", level, "] ", msg, "\n"], + time_offset => TimeOffset + }} +). + +-define(FILTERS, [{drop_progress_reports, {fun logger_filters:progress/2, stop}}]). +-define(LOG_CONFIG, #{ + burst_limit_enable => true, + burst_limit_max_count => 10000, + burst_limit_window_time => 1000, + drop_mode_qlen => 3000, + flush_qlen => 8000, + overload_kill_enable => true, + overload_kill_mem_size => 31457280, + overload_kill_qlen => 20000, + overload_kill_restart_after => 5000, + sync_mode_qlen => 100 +}). outdated_log_test() -> + validate_log(?OUTDATED_LOG_CONF). + +validate_log(Conf) -> BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), - Conf0 = <>, + Conf0 = <>, {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), ConfList = hocon_tconf:generate(emqx_conf_schema, ConfMap0), - ct:pal("fff:~p", [ConfList]), - Log = proplists:get_value(log, ConfList), - Console = proplists:get_value(console, Log), - File = proplists:get_value(file, Log), - ?assertEqual(1, Console, {Console, File}), - ok. + Kernel = proplists:get_value(kernel, ConfList), --define(NEW_LOG_CONF, - "" - "\n" - "console {\n" - " enable = true\n" - " formatter = text\n" - " level = warning\n" - " time_offset = system\n" - "}\n" - "file {\n" - " enable = true\n" - " file = \"log/emqx.log\"\n" - " formatter = text\n" - " level = warning\n" - " rotation {count = 10, enable = true}\n" - " time_offset = \"+01:00\"\n" - " }\n" - "file_handlers.default {\n" - " enable = false,\n" - " file = \"log/file_handlres_emqx.log\"\n" - " }\n" - "}\n" - " " - "" + ?assertEqual(silent, proplists:get_value(error_logger, Kernel)), + ?assertEqual(debug, proplists:get_value(logger_level, Kernel)), + Loggers = proplists:get_value(logger, Kernel), + FileHandler = lists:keyfind(logger_disk_log_h, 3, Loggers), + ?assertEqual( + {handler, default, logger_disk_log_h, #{ + config => ?LOG_CONFIG#{ + type => wrap, + file => "log/my-emqx.log", + max_no_bytes => 1073741824, + max_no_files => 20 + }, + filesync_repeat_interval => no_repeat, + filters => ?FILTERS, + formatter => ?FORMATTER("+01:00"), + level => debug + }}, + FileHandler + ), + ConsoleHandler = lists:keyfind(logger_std_h, 3, Loggers), + ?assertEqual( + {handler, console, logger_std_h, #{ + config => ?LOG_CONFIG#{type => standard_io}, + filters => ?FILTERS, + formatter => ?FORMATTER("+02:00"), + level => warning + }}, + ConsoleHandler + ). + +%% erlfmt-ignore +-define(KERNEL_LOG_CONF, + """ + log.console { + enable = true + formatter = text + level = warning + time_offset = \"+02:00\" + } + log.file { + enable = false + file = \"log/emqx.log\" + formatter = text + level = debug + rotation_count = 20 + rotation_size = \"1024MB\" + time_offset = \"+01:00\" + } + log.file_handlers.default { + enable = true + file = \"log/my-emqx.log\" + } + """ ). +log_test() -> + validate_log(?KERNEL_LOG_CONF). + %% erlfmt-ignore -define(BASE_AUTHN_ARRAY, """ @@ -181,38 +234,50 @@ authn_validations_test() -> OKHttps = to_bin(?BASE_AUTHN_ARRAY, [post, true, <<"https://127.0.0.1:8080">>]), Conf0 = <>, {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), - ?assert(is_list(hocon_tconf:generate(emqx_conf_schema, ConfMap0))), + {_, Res0} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap0, #{format => richmap}), + Headers0 = authentication_headers(Res0), + ?assertEqual(<<"application/json">>, maps:get(<<"content-type">>, Headers0)), + %% accept from converter + ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers0)), OKHttp = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"http://127.0.0.1:8080">>]), Conf1 = <>, {ok, ConfMap1} = hocon:binary(Conf1, #{format => richmap}), - ?assert(is_list(hocon_tconf:generate(emqx_conf_schema, ConfMap1))), + {_, Res1} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap1, #{format => richmap}), + Headers1 = authentication_headers(Res1), + ?assertEqual(<<"application/json">>, maps:get(<<"content-type">>, Headers1), Headers1), + ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers1), Headers1), DisableSSLWithHttps = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"https://127.0.0.1:8080">>]), Conf2 = <>, {ok, ConfMap2} = hocon:binary(Conf2, #{format => richmap}), ?assertThrow( ?ERROR(check_http_ssl_opts), - hocon_tconf:generate(emqx_conf_schema, ConfMap2) + hocon_tconf:map_translate(emqx_conf_schema, ConfMap2, #{format => richmap}) ), BadHeader = to_bin(?BASE_AUTHN_ARRAY, [get, true, <<"https://127.0.0.1:8080">>]), Conf3 = <>, {ok, ConfMap3} = hocon:binary(Conf3, #{format => richmap}), - ?assertThrow( - ?ERROR(check_http_headers), - hocon_tconf:generate(emqx_conf_schema, ConfMap3) - ), + {_, Res3} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap3, #{format => richmap}), + Headers3 = authentication_headers(Res3), + %% remove the content-type header when get method + ?assertEqual(false, maps:is_key(<<"content-type">>, Headers3), Headers3), + ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers3), Headers3), BadHeaderWithTuple = binary:replace(BadHeader, [<<"[">>, <<"]">>], <<"">>, [global]), Conf4 = <>, {ok, ConfMap4} = hocon:binary(Conf4, #{format => richmap}), - ?assertThrow( - ?ERROR(check_http_headers), - hocon_tconf:generate(emqx_conf_schema, ConfMap4) - ), + {_, Res4} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap4, #{}), + Headers4 = authentication_headers(Res4), + ?assertEqual(false, maps:is_key(<<"content-type">>, Headers4), Headers4), + ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers4), Headers4), ok. +authentication_headers(Conf) -> + [#{<<"headers">> := Headers}] = hocon_maps:get("authentication", Conf), + Headers. + doc_gen_test() -> %% the json file too large to encode. { From ad111a27f90d2165ad342513f64ef0acd6d5bfc8 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 8 May 2023 17:40:41 +0800 Subject: [PATCH 029/144] feat: alias log.file.to to log.file.file --- apps/emqx/rebar.config | 2 +- apps/emqx/src/config/emqx_config_logger.erl | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- .../emqx_conf/test/emqx_conf_schema_tests.erl | 44 ++++++++++++++++++- mix.exs | 2 +- rebar.config | 2 +- 6 files changed, 48 insertions(+), 6 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 8e51a4079..ee33de899 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.1"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.5"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.6"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl index 901056c0f..983f50700 100644 --- a/apps/emqx/src/config/emqx_config_logger.erl +++ b/apps/emqx/src/config/emqx_config_logger.erl @@ -151,7 +151,7 @@ tr_file_handler({HandlerName, SubConf}) -> level => conf_get("level", SubConf), config => (log_handler_conf(SubConf))#{ type => wrap, - file => conf_get("sink_to", SubConf), + file => conf_get("to", SubConf), max_no_files => conf_get("rotation_count", SubConf), max_no_bytes => conf_get("rotation_size", SubConf) }, diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 0e41d598c..97efa67cc 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -891,7 +891,7 @@ fields("console_handler") -> log_handler_common_confs(console); fields("log_file_handler") -> [ - {"sink_to", + {"to", sc( file(), #{ diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index b0eea9a07..3577edeab 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -180,7 +180,7 @@ validate_log(Conf) -> } log.file { enable = false - file = \"log/emqx.log\" + file = \"log/xx-emqx.log\" formatter = text level = debug rotation_count = 20 @@ -197,6 +197,48 @@ validate_log(Conf) -> log_test() -> validate_log(?KERNEL_LOG_CONF). +%% erlfmt-ignore +log_rotation_count_limit_test() -> + Format = + """ + log.file { + enable = true + to = \"log/emqx.log\" + formatter = text + level = debug + rotation = {count = ~w} + rotation_size = \"1024MB\" + } + """, + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + lists:foreach(fun({Conf, Count}) -> + Conf0 = <>, + {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), + ConfList = hocon_tconf:generate(emqx_conf_schema, ConfMap0), + Kernel = proplists:get_value(kernel, ConfList), + Loggers = proplists:get_value(logger, Kernel), + ?assertMatch( + {handler, default, logger_disk_log_h, #{ + config := #{max_no_files := Count} + }}, + lists:keyfind(logger_disk_log_h, 3, Loggers) + ) + end, + [{to_bin(Format, [1]), 1}, {to_bin(Format, [128]), 128}]), + lists:foreach(fun({Conf, Count}) -> + Conf0 = <>, + {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), + ?assertThrow({emqx_conf_schema, + [#{kind := validation_error, + mismatches := #{"handler_name" := + #{kind := validation_error, + path := "log.file.default.rotation_count", + reason := #{expected_type := "1..128"}, + value := Count} + }}]}, + hocon_tconf:generate(emqx_conf_schema, ConfMap0)) + end, [{to_bin(Format, [0]), 0}, {to_bin(Format, [129]), 129}]). + %% erlfmt-ignore -define(BASE_AUTHN_ARRAY, """ diff --git a/mix.exs b/mix.exs index cdacb2986..b1e36effa 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.39.5", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.39.6", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 738b3ffb0..bbabd7849 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.5"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.6"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From c901f3a9d467617c4050ee52b71c8ffba5b7d8ae Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 9 May 2023 11:56:12 +0800 Subject: [PATCH 030/144] test: nodes's log test failed --- apps/emqx_management/src/emqx_mgmt.erl | 2 +- apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 0b91817f0..9553730ec 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -151,7 +151,7 @@ log_path() -> Configs = logger:get_handler_config(), case get_log_path(Configs) of undefined -> - <<"log.file_handler.default.enable is false, not logging to file.">>; + <<"log.file.enable is false, not logging to file.">>; Path -> iolist_to_binary(filename:join(RootDir, Path)) end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl index 1f14d075e..b356bf905 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl @@ -34,8 +34,8 @@ init_per_testcase(t_log_path, Config) -> emqx_config_logger:add_handler(), Log = emqx_conf:get_raw([log], #{}), File = "log/emqx-test.log", - Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), - Log2 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), + Log1 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"enable">>], Log, true), + Log2 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"to">>], Log1, File), {ok, #{}} = emqx_conf:update([log], Log2, #{rawconf_with_defaults => true}), Config; init_per_testcase(_, Config) -> @@ -43,7 +43,7 @@ init_per_testcase(_, Config) -> end_per_testcase(t_log_path, Config) -> Log = emqx_conf:get_raw([log], #{}), - Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, false), + Log1 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"enable">>], Log, false), {ok, #{}} = emqx_conf:update([log], Log1, #{rawconf_with_defaults => true}), emqx_config_logger:remove_handler(), Config; From ac6c8d840f720a28f0366dbba872efe8cbb7e0c1 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 9 May 2023 12:16:45 +0800 Subject: [PATCH 031/144] test: dashboard_listener_test crash --- apps/emqx_dashboard/test/emqx_dashboard_listener_SUITE.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_dashboard/test/emqx_dashboard_listener_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_listener_SUITE.erl index 7f28841fc..1bc463b1f 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_listener_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_listener_SUITE.erl @@ -25,6 +25,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + emqx_common_test_helpers:load_config(emqx_dashboard_schema, <<"dashboard {}">>), emqx_mgmt_api_test_util:init_suite([emqx_conf]), ok = change_i18n_lang(en), Config. From a119100a0ed8c20a6c8bba936f7a62e5599a8461 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 9 May 2023 12:00:31 +0800 Subject: [PATCH 032/144] refactor(rocketmq): move rocketmq bridge into its own app --- .ci/docker-compose-file/docker-compose-rocketmq.yaml | 4 ++-- apps/emqx_bridge_rocketmq/docker-ct | 2 ++ apps/emqx_bridge_rocketmq/rebar.config | 8 ++++++++ .../emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src | 2 +- .../emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.erl | 6 +++--- .../src/emqx_bridge_rocketmq_connector.erl | 4 ++-- .../test/emqx_bridge_rocketmq_SUITE.erl | 2 +- lib-ee/emqx_ee_bridge/docker-ct | 1 - lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src | 3 ++- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 8 ++++---- lib-ee/emqx_ee_connector/rebar.config | 1 - lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src | 3 +-- ...e_bridge_rocketmq.hocon => emqx_bridge_rocketmq.hocon} | 2 +- ...ocketmq.hocon => emqx_bridge_rocketmq_connector.hocon} | 2 +- ...e_bridge_rocketmq.hocon => emqx_bridge_rocketmq.hocon} | 2 +- ...ocketmq.hocon => emqx_bridge_rocketmq_connector.hocon} | 2 +- 16 files changed, 30 insertions(+), 22 deletions(-) create mode 100644 apps/emqx_bridge_rocketmq/docker-ct create mode 100644 apps/emqx_bridge_rocketmq/rebar.config rename lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl => apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.erl (94%) rename lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl => apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl (98%) rename lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl => apps/emqx_bridge_rocketmq/test/emqx_bridge_rocketmq_SUITE.erl (99%) rename rel/i18n/{emqx_ee_bridge_rocketmq.hocon => emqx_bridge_rocketmq.hocon} (97%) rename rel/i18n/{emqx_ee_connector_rocketmq.hocon => emqx_bridge_rocketmq_connector.hocon} (96%) rename rel/i18n/zh/{emqx_ee_bridge_rocketmq.hocon => emqx_bridge_rocketmq.hocon} (97%) rename rel/i18n/zh/{emqx_ee_connector_rocketmq.hocon => emqx_bridge_rocketmq_connector.hocon} (96%) diff --git a/.ci/docker-compose-file/docker-compose-rocketmq.yaml b/.ci/docker-compose-file/docker-compose-rocketmq.yaml index 3c872a7c2..7e5a2e42e 100644 --- a/.ci/docker-compose-file/docker-compose-rocketmq.yaml +++ b/.ci/docker-compose-file/docker-compose-rocketmq.yaml @@ -25,8 +25,8 @@ services: - ./rocketmq/conf/broker.conf:/etc/rocketmq/broker.conf environment: NAMESRV_ADDR: "rocketmq_namesrv:9876" - JAVA_OPTS: " -Duser.home=/opt" - JAVA_OPT_EXT: "-server -Xms1024m -Xmx1024m -Xmn1024m" + JAVA_OPTS: " -Duser.home=/opt -Drocketmq.broker.diskSpaceWarningLevelRatio=0.99" + JAVA_OPT_EXT: "-server -Xms512m -Xmx512m -Xmn512m" command: ./mqbroker -c /etc/rocketmq/broker.conf depends_on: - mqnamesrv diff --git a/apps/emqx_bridge_rocketmq/docker-ct b/apps/emqx_bridge_rocketmq/docker-ct new file mode 100644 index 000000000..463a9eb66 --- /dev/null +++ b/apps/emqx_bridge_rocketmq/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +rocketmq diff --git a/apps/emqx_bridge_rocketmq/rebar.config b/apps/emqx_bridge_rocketmq/rebar.config new file mode 100644 index 000000000..1af22f108 --- /dev/null +++ b/apps/emqx_bridge_rocketmq/rebar.config @@ -0,0 +1,8 @@ +{erl_opts, [debug_info]}. + +{deps, [ + {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}}, + {emqx_bridge, {path, "../../apps/emqx_bridge"}} +]}. diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src index e1916034c..1ad9427df 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src @@ -2,7 +2,7 @@ {description, "EMQX Enterprise RocketMQ Bridge"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, rocketmq]}, {env, []}, {modules, []}, {links, []} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.erl similarity index 94% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl rename to apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.erl index 28b94a1a4..a4a942d0e 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_rocketmq). +-module(emqx_bridge_rocketmq). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -82,7 +82,7 @@ fields("config") -> #{desc => ?DESC("local_topic"), required => false} )} ] ++ emqx_resource_schema:fields("resource_opts") ++ - (emqx_ee_connector_rocketmq:fields(config) -- + (emqx_bridge_rocketmq_connector:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); fields("post") -> [type_field(), name_field() | fields("config")]; diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl similarity index 98% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl rename to apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl index 52b49a8a9..a3da57147 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl @@ -1,8 +1,8 @@ %-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_rocketmq). +-module(emqx_bridge_rocketmq_connector). -behaviour(emqx_resource). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl b/apps/emqx_bridge_rocketmq/test/emqx_bridge_rocketmq_SUITE.erl similarity index 99% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl rename to apps/emqx_bridge_rocketmq/test/emqx_bridge_rocketmq_SUITE.erl index 33a83d2d8..90047e577 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl +++ b/apps/emqx_bridge_rocketmq/test/emqx_bridge_rocketmq_SUITE.erl @@ -2,7 +2,7 @@ % Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_rocketmq_SUITE). +-module(emqx_bridge_rocketmq_SUITE). -compile(nowarn_export_all). -compile(export_all). diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index 469271541..37a5c9765 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -9,4 +9,3 @@ pgsql tdengine clickhouse dynamo -rocketmq diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index d0317cbc9..39f6f2043 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -12,7 +12,8 @@ emqx_bridge_cassandra, emqx_bridge_opents, emqx_bridge_pulsar, - emqx_bridge_sqlserver + emqx_bridge_sqlserver, + emqx_bridge_rocketmq ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 8581f79b3..121ba5c0e 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -34,7 +34,7 @@ api_schemas(Method) -> ref(emqx_ee_bridge_tdengine, Method), ref(emqx_ee_bridge_clickhouse, Method), ref(emqx_ee_bridge_dynamo, Method), - ref(emqx_ee_bridge_rocketmq, Method), + ref(emqx_bridge_rocketmq, Method), ref(emqx_bridge_sqlserver, Method), ref(emqx_bridge_opents, Method), ref(emqx_bridge_pulsar, Method ++ "_producer"), @@ -58,7 +58,7 @@ schema_modules() -> emqx_ee_bridge_tdengine, emqx_ee_bridge_clickhouse, emqx_ee_bridge_dynamo, - emqx_ee_bridge_rocketmq, + emqx_bridge_rocketmq, emqx_bridge_sqlserver, emqx_bridge_opents, emqx_bridge_pulsar, @@ -101,7 +101,7 @@ resource_type(matrix) -> emqx_connector_pgsql; resource_type(tdengine) -> emqx_ee_connector_tdengine; resource_type(clickhouse) -> emqx_ee_connector_clickhouse; resource_type(dynamo) -> emqx_ee_connector_dynamo; -resource_type(rocketmq) -> emqx_ee_connector_rocketmq; +resource_type(rocketmq) -> emqx_bridge_rocketmq_connector; resource_type(sqlserver) -> emqx_bridge_sqlserver_connector; resource_type(opents) -> emqx_bridge_opents_connector; resource_type(pulsar_producer) -> emqx_bridge_pulsar_impl_producer; @@ -152,7 +152,7 @@ fields(bridges) -> )}, {rocketmq, mk( - hoconsc:map(name, ref(emqx_ee_bridge_rocketmq, "config")), + hoconsc:map(name, ref(emqx_bridge_rocketmq, "config")), #{ desc => <<"RocketMQ Bridge Config">>, required => false diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 352c54629..a61ed9630 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -5,7 +5,6 @@ {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}}, - {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, {emqx, {path, "../../apps/emqx"}}, {emqx_utils, {path, "../../apps/emqx_utils"}} ]}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 68e36f48a..d02995975 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -10,8 +10,7 @@ influxdb, tdengine, clickhouse, - erlcloud, - rocketmq + erlcloud ]}, {env, []}, {modules, []}, diff --git a/rel/i18n/emqx_ee_bridge_rocketmq.hocon b/rel/i18n/emqx_bridge_rocketmq.hocon similarity index 97% rename from rel/i18n/emqx_ee_bridge_rocketmq.hocon rename to rel/i18n/emqx_bridge_rocketmq.hocon index e079220b6..ac5deb757 100644 --- a/rel/i18n/emqx_ee_bridge_rocketmq.hocon +++ b/rel/i18n/emqx_bridge_rocketmq.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_rocketmq { +emqx_bridge_rocketmq { config_enable.desc: """Enable or disable this bridge""" diff --git a/rel/i18n/emqx_ee_connector_rocketmq.hocon b/rel/i18n/emqx_bridge_rocketmq_connector.hocon similarity index 96% rename from rel/i18n/emqx_ee_connector_rocketmq.hocon rename to rel/i18n/emqx_bridge_rocketmq_connector.hocon index d3d59a389..b13e015c2 100644 --- a/rel/i18n/emqx_ee_connector_rocketmq.hocon +++ b/rel/i18n/emqx_bridge_rocketmq_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_rocketmq { +emqx_bridge_rocketmq_connector { access_key.desc: """RocketMQ server `accessKey`.""" diff --git a/rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon b/rel/i18n/zh/emqx_bridge_rocketmq.hocon similarity index 97% rename from rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon rename to rel/i18n/zh/emqx_bridge_rocketmq.hocon index 445a54232..75d2588de 100644 --- a/rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon +++ b/rel/i18n/zh/emqx_bridge_rocketmq.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_rocketmq { +emqx_bridge_rocketmq { config_enable.desc: """启用/禁用桥接""" diff --git a/rel/i18n/zh/emqx_ee_connector_rocketmq.hocon b/rel/i18n/zh/emqx_bridge_rocketmq_connector.hocon similarity index 96% rename from rel/i18n/zh/emqx_ee_connector_rocketmq.hocon rename to rel/i18n/zh/emqx_bridge_rocketmq_connector.hocon index 58a1f7ddb..abc7bcdce 100644 --- a/rel/i18n/zh/emqx_ee_connector_rocketmq.hocon +++ b/rel/i18n/zh/emqx_bridge_rocketmq_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_rocketmq { +emqx_bridge_rocket_connector { access_key.desc: """RocketMQ 服务器的 `accessKey`。""" From 12491e08db486a5db9fff51498e7402b748a317a Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 9 May 2023 15:02:12 +0800 Subject: [PATCH 033/144] chore: bump version && update changes --- apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src | 2 +- changes/ee/feat-10648.en.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/ee/feat-10648.en.md diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src index 1ad9427df..51189d174 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_rocketmq, [ {description, "EMQX Enterprise RocketMQ Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [kernel, stdlib, rocketmq]}, {env, []}, diff --git a/changes/ee/feat-10648.en.md b/changes/ee/feat-10648.en.md new file mode 100644 index 000000000..4524155d8 --- /dev/null +++ b/changes/ee/feat-10648.en.md @@ -0,0 +1 @@ +Refactor the directory structure of the RocketMQ data bridge. From 9af8375773334b494825162b6622b6720d0a17ec Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 9 May 2023 10:59:39 +0200 Subject: [PATCH 034/144] docs: improve rule engine labels and descriptions Fixes: https://emqx.atlassian.net/browse/EMQX-9645 --- rel/i18n/emqx_rule_api_schema.hocon | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/rel/i18n/emqx_rule_api_schema.hocon b/rel/i18n/emqx_rule_api_schema.hocon index 29ecaa18e..0289f53ab 100644 --- a/rel/i18n/emqx_rule_api_schema.hocon +++ b/rel/i18n/emqx_rule_api_schema.hocon @@ -13,16 +13,16 @@ event_payload.label: """Message Payload""" metrics_actions_failed_out_of_service.desc: -"""How much times the rule failed to call actions due to the action is out of service. For example, a bridge is disabled or stopped.""" +"""How many times the rule has failed to call actions due to the action is out of service. For example, a bridge is disabled or stopped.""" metrics_actions_failed_out_of_service.label: """Fail Action""" metrics_actions_failed_unknown.desc: -"""How much times the rule failed to call actions due to to an unknown error.""" +"""The number of action failures that have occurred due to unanticipated reasons. For more information on these errors, please refer to the EMQX log file.""" metrics_actions_failed_unknown.label: -"""Fail Action""" +"""Unknown Failures""" event_server.desc: """The IP address (or hostname) and port of the MQTT broker, in IP:Port format""" @@ -31,7 +31,7 @@ event_server.label: """Server IP And Port""" metrics_actions_total.desc: -"""How much times the actions are called by the rule. This value may several times of 'matched', depending on the number of the actions of the rule.""" +"""How many times the actions are called by the rule. This value may several times of 'matched', depending on the number of the actions of the rule.""" metrics_actions_total.label: """Action Total""" @@ -55,7 +55,7 @@ event_peername.label: """IP Address And Port""" metrics_sql_passed.desc: -"""How much times the SQL is passed""" +"""How many times the SQL is passed""" metrics_sql_passed.label: """SQL Passed""" @@ -91,7 +91,7 @@ event_connected_at.label: """Connected Time""" metrics_sql_failed_exception.desc: -"""How much times the SQL is failed due to exceptions. This may because of a crash when calling a SQL function, or trying to do arithmetic operation on undefined variables""" +"""How many times the SQL is failed due to exceptions. This may because of a crash when calling a SQL function, or trying to do arithmetic operation on undefined variables""" metrics_sql_failed_exception.label: """SQL Exception""" @@ -181,7 +181,7 @@ event_expiry_interval.label: """Expiry Interval""" metrics_sql_matched.desc: -"""How much times the FROM clause of the SQL is matched.""" +"""How many times the FROM clause of the SQL is matched.""" metrics_sql_matched.label: """Matched""" @@ -193,13 +193,13 @@ event_clientid.label: """Client ID""" metrics_actions_success.desc: -"""How much times the rule success to call the actions.""" +"""How many times the rule successided to call the actions.""" metrics_actions_success.label: """Success Action""" metrics_actions_failed.desc: -"""How much times the rule failed to call the actions.""" +"""How many times the rule failed to call the actions.""" metrics_actions_failed.label: """Failed Action""" @@ -241,13 +241,13 @@ event_authz_source.label: """Auth Source""" metrics_sql_failed_unknown.desc: -"""How much times the SQL is failed due to an unknown error.""" +"""How many times the SQL is failed due to an unknown error.""" metrics_sql_failed_unknown.label: """SQL Unknown Error""" metrics_sql_failed.desc: -"""How much times the SQL is failed""" +"""How many times the SQL statement has failed""" metrics_sql_failed.label: """SQL Failed""" From 428cc45c69c8205379f579cb2dd673538083b5e2 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 9 May 2023 17:50:06 +0800 Subject: [PATCH 035/144] fix: filter disable file handler --- apps/emqx/src/config/emqx_config_logger.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl index 983f50700..a7bf54aee 100644 --- a/apps/emqx/src/config/emqx_config_logger.erl +++ b/apps/emqx/src/config/emqx_config_logger.erl @@ -161,7 +161,12 @@ tr_file_handler({HandlerName, SubConf}) -> }}. logger_file_handlers(Conf) -> - maps:to_list(conf_get("log.file", Conf, #{})). + lists:filter( + fun({_Name, Handler}) -> + conf_get("enable", Handler, false) + end, + maps:to_list(conf_get("log.file", Conf, #{})) + ). conf_get(Key, Conf) -> emqx_schema:conf_get(Key, Conf). conf_get(Key, Conf, Default) -> emqx_schema:conf_get(Key, Conf, Default). From a45ef4d94df72feef8c927ac6eb42bad690229d2 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 9 May 2023 13:50:36 +0200 Subject: [PATCH 036/144] ci: remove unnecessary matrix dimensions --- .github/workflows/build_packages_cron.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index 7f6773f4a..b7ef44a79 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -24,9 +24,6 @@ jobs: profile: - ['emqx', 'master'] - ['emqx-enterprise', 'release-50'] - branch: - - master - - release-50 otp: - 24.3.4.2-3 arch: From 70cf1533dba7438832837e16cc2945f6bc209439 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 17 Apr 2023 15:26:54 +0200 Subject: [PATCH 037/144] feat: add RabbitMQ bridge --- .../docker-compose-rabbitmq.yaml | 17 + apps/emqx/src/emqx_schema.erl | 5 +- apps/emqx/test/emqx_bpapi_static_checks.erl | 4 +- apps/emqx_bridge_rabbitmq/BSL.txt | 94 +++ apps/emqx_bridge_rabbitmq/README.md | 46 ++ apps/emqx_bridge_rabbitmq/docker-ct | 1 + apps/emqx_bridge_rabbitmq/rebar.config | 33 ++ .../src/emqx_bridge_rabbitmq.app.src | 9 + .../src/emqx_bridge_rabbitmq.erl | 124 ++++ .../src/emqx_bridge_rabbitmq_connector.erl | 548 ++++++++++++++++++ .../test/emqx_bridge_rabbitmq_SUITE.erl | 371 ++++++++++++ .../emqx_bridge_rabbitmq_connector_SUITE.erl | 232 ++++++++ .../src/emqx_dashboard_swagger.erl | 2 + apps/emqx_resource/src/emqx_resource.erl | 3 + changes/ee/feat-10534.md | 1 + .../emqx_ee_bridge/src/emqx_ee_bridge.app.src | 3 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 23 +- mix.exs | 23 +- rebar.config.erl | 4 +- rel/i18n/emqx_bridge_rabbitmq.hocon | 34 ++ rel/i18n/emqx_bridge_rabbitmq_connector.hocon | 100 ++++ scripts/check-elixir-deps-discrepancies.exs | 3 + scripts/ct/run.sh | 3 + 23 files changed, 1671 insertions(+), 12 deletions(-) create mode 100644 .ci/docker-compose-file/docker-compose-rabbitmq.yaml create mode 100644 apps/emqx_bridge_rabbitmq/BSL.txt create mode 100644 apps/emqx_bridge_rabbitmq/README.md create mode 100644 apps/emqx_bridge_rabbitmq/docker-ct create mode 100644 apps/emqx_bridge_rabbitmq/rebar.config create mode 100644 apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src create mode 100644 apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.erl create mode 100644 apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl create mode 100644 apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_SUITE.erl create mode 100644 apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl create mode 100644 changes/ee/feat-10534.md create mode 100644 rel/i18n/emqx_bridge_rabbitmq.hocon create mode 100644 rel/i18n/emqx_bridge_rabbitmq_connector.hocon diff --git a/.ci/docker-compose-file/docker-compose-rabbitmq.yaml b/.ci/docker-compose-file/docker-compose-rabbitmq.yaml new file mode 100644 index 000000000..76df9d24c --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-rabbitmq.yaml @@ -0,0 +1,17 @@ +version: '3.9' + +services: + rabbitmq: + container_name: rabbitmq + image: rabbitmq:3.11-management + + restart: always + expose: + - "15672" + - "5672" + # We don't want to take ports from the host + # ports: + # - "15672:15672" + # - "5672:5672" + networks: + - emqx_bridge diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index f094de33b..cba67aca4 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -42,7 +42,7 @@ -type bar_separated_list() :: list(). -type ip_port() :: tuple() | integer(). -type cipher() :: map(). --type port_number() :: 1..65536. +-type port_number() :: 1..65535. -type server_parse_option() :: #{ default_port => port_number(), no_port => boolean(), @@ -135,7 +135,8 @@ cipher/0, comma_separated_atoms/0, url/0, - json_binary/0 + json_binary/0, + port_number/0 ]). -export([namespace/0, roots/0, roots/1, fields/1, desc/1, tags/0]). diff --git a/apps/emqx/test/emqx_bpapi_static_checks.erl b/apps/emqx/test/emqx_bpapi_static_checks.erl index 142750cac..34ff149c1 100644 --- a/apps/emqx/test/emqx_bpapi_static_checks.erl +++ b/apps/emqx/test/emqx_bpapi_static_checks.erl @@ -47,7 +47,9 @@ -type param_types() :: #{emqx_bpapi:var_name() => _Type}. %% Applications and modules we wish to ignore in the analysis: --define(IGNORED_APPS, "gen_rpc, recon, redbug, observer_cli, snabbkaffe, ekka, mria"). +-define(IGNORED_APPS, + "gen_rpc, recon, redbug, observer_cli, snabbkaffe, ekka, mria, amqp_client, rabbit_common" +). -define(IGNORED_MODULES, "emqx_rpc"). %% List of known RPC backend modules: -define(RPC_MODULES, "gen_rpc, erpc, rpc, emqx_rpc"). diff --git a/apps/emqx_bridge_rabbitmq/BSL.txt b/apps/emqx_bridge_rabbitmq/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_rabbitmq/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_rabbitmq/README.md b/apps/emqx_bridge_rabbitmq/README.md new file mode 100644 index 000000000..420a9e048 --- /dev/null +++ b/apps/emqx_bridge_rabbitmq/README.md @@ -0,0 +1,46 @@ +# EMQX RabbitMQ Bridge + +[RabbitMQ](https://www.rabbitmq.com/) is a powerful, open-source message broker +that facilitates asynchronous communication between different components of an +application. Built on the Advanced Message Queuing Protocol (AMQP), RabbitMQ +enables the reliable transmission of messages by decoupling the sender and +receiver components. This separation allows for increased scalability, +robustness, and flexibility in application architecture. + +RabbitMQ is commonly used for a wide range of purposes, such as distributing +tasks among multiple workers, enabling event-driven architectures, and +implementing publish-subscribe patterns. It is a popular choice for +microservices, distributed systems, and real-time applications, providing an +efficient way to handle varying workloads and ensuring message delivery in +complex environments. + +This application is used to connect EMQX and RabbitMQ. User can create a rule +and easily ingest IoT data into RabbitMQ by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to the [RabbitMQ bridge documentation](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-rabbitmq.html) + for how to use EMQX dashboard to ingest IoT data into RabbitMQ. +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for an introduction to the EMQX rules engine. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). + diff --git a/apps/emqx_bridge_rabbitmq/docker-ct b/apps/emqx_bridge_rabbitmq/docker-ct new file mode 100644 index 000000000..5232abf91 --- /dev/null +++ b/apps/emqx_bridge_rabbitmq/docker-ct @@ -0,0 +1 @@ +rabbitmq diff --git a/apps/emqx_bridge_rabbitmq/rebar.config b/apps/emqx_bridge_rabbitmq/rebar.config new file mode 100644 index 000000000..3f1c5d3fc --- /dev/null +++ b/apps/emqx_bridge_rabbitmq/rebar.config @@ -0,0 +1,33 @@ +%% -*- mode: erlang; -*- +{erl_opts, [debug_info]}. +{deps, [ + %% The following two are dependencies of rabbit_common + {thoas, {git, "https://github.com/emqx/thoas.git", {tag, "v1.0.0"}}} + , {credentials_obfuscation, {git, "https://github.com/emqx/credentials-obfuscation.git", {tag, "v3.2.0"}}} + %% The v3.11.13_with_app_src tag, employed in the next two dependencies, + %% represents a fork of the official RabbitMQ v3.11.13 tag. This fork diverges + %% from the official version as it includes app and hrl files + %% generated by make files in subdirectories deps/rabbit_common and + %% deps/amqp_client (app files are also relocated from the ebin to the src + %% directory). This modification ensures compatibility with rebar3, as + %% rabbit_common and amqp_client utilize the erlang.mk build tool. + %% Similar changes are probably needed when upgrading to newer versions + %% of rabbit_common and amqp_client. There are hex packages for rabbit_common and + %% amqp_client, but they are not used here as we don't want to depend on + %% packages that we don't have control over. + , {rabbit_common, {git_subdir, + "https://github.com/emqx/rabbitmq-server.git", + {tag, "v3.11.13-emqx"}, + "deps/rabbit_common"}} + , {amqp_client, {git_subdir, + "https://github.com/emqx/rabbitmq-server.git", + {tag, "v3.11.13-emqx"}, + "deps/amqp_client"}} + , {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + {apps, [emqx_bridge_rabbitmq]} +]}. diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src new file mode 100644 index 000000000..36f47aaf6 --- /dev/null +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_rabbitmq, [ + {description, "EMQX Enterprise RabbitMQ Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib, ecql, rabbit_common, amqp_client]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.erl new file mode 100644 index 000000000..2039fadf6 --- /dev/null +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.erl @@ -0,0 +1,124 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_rabbitmq). + +-include_lib("emqx_bridge/include/emqx_bridge.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([ + conn_bridge_examples/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +%% ------------------------------------------------------------------------------------------------- +%% Callback used by HTTP API +%% ------------------------------------------------------------------------------------------------- + +conn_bridge_examples(Method) -> + [ + #{ + <<"rabbitmq">> => #{ + summary => <<"RabbitMQ Bridge">>, + value => values(Method, "rabbitmq") + } + } + ]. + +values(_Method, Type) -> + #{ + enable => true, + type => Type, + name => <<"foo">>, + server => <<"localhost">>, + port => 5672, + username => <<"guest">>, + password => <<"******">>, + pool_size => 8, + timeout => 5, + virtual_host => <<"/">>, + heartbeat => <<"30s">>, + auto_reconnect => <<"2s">>, + exchange => <<"messages">>, + exchange_type => <<"topic">>, + routing_key => <<"my_routing_key">>, + durable => false, + payload_template => <<"">>, + resource_opts => #{ + worker_pool_size => 8, + health_check_interval => ?HEALTHCHECK_INTERVAL_RAW, + auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW, + batch_size => ?DEFAULT_BATCH_SIZE, + batch_time => ?DEFAULT_BATCH_TIME, + query_mode => async, + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES + } + }. + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +%% ------------------------------------------------------------------------------------------------- + +namespace() -> "bridge_rabbitmq". + +roots() -> []. + +fields("config") -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {local_topic, + mk( + binary(), + #{desc => ?DESC("local_topic"), default => undefined} + )}, + {resource_opts, + mk( + ref(?MODULE, "creation_opts"), + #{ + required => false, + default => #{}, + desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) + } + )} + ] ++ + emqx_bridge_rabbitmq_connector:fields(config); +fields("creation_opts") -> + emqx_resource_schema:fields("creation_opts"); +fields("post") -> + fields("post", clickhouse); +fields("put") -> + fields("config"); +fields("get") -> + emqx_bridge_schema:status_fields() ++ fields("post"). + +fields("post", Type) -> + [type_field(Type), name_field() | fields("config")]. + +desc("config") -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for RabbitMQ using `", string:to_upper(Method), "` method."]; +desc("creation_opts" = Name) -> + emqx_resource_schema:desc(Name); +desc(_) -> + undefined. + +%% ------------------------------------------------------------------------------------------------- +%% internal +%% ------------------------------------------------------------------------------------------------- + +type_field(Type) -> + {type, mk(enum([Type]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl new file mode 100644 index 000000000..6a7a68591 --- /dev/null +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl @@ -0,0 +1,548 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_rabbitmq_connector). + +-include_lib("emqx_connector/include/emqx_connector.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% Needed to create RabbitMQ connection +-include_lib("amqp_client/include/amqp_client.hrl"). + +-behaviour(emqx_resource). +-behaviour(hocon_schema). +-behaviour(ecpool_worker). + +%% hocon_schema callbacks +-export([roots/0, fields/1]). + +%% HTTP API callbacks +-export([values/1]). + +%% emqx_resource callbacks +-export([ + %% Required callbacks + on_start/2, + on_stop/2, + callback_mode/0, + %% Optional callbacks + on_get_status/2, + on_query/3, + is_buffer_supported/0, + on_batch_query/3 +]). + +%% callbacks for ecpool_worker +-export([connect/1]). + +%% Internal callbacks +-export([publish_messages/3]). + +roots() -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]. + +fields(config) -> + [ + {server, + hoconsc:mk( + typerefl:binary(), + #{ + default => <<"localhost">>, + desc => ?DESC("server") + } + )}, + {port, + hoconsc:mk( + emqx_schema:port_number(), + #{ + default => 5672, + desc => ?DESC("server") + } + )}, + {username, + hoconsc:mk( + typerefl:binary(), + #{ + required => true, + desc => ?DESC("username") + } + )}, + {password, + hoconsc:mk( + typerefl:binary(), + #{ + required => true, + desc => ?DESC("password") + } + )}, + {pool_size, + hoconsc:mk( + typerefl:pos_integer(), + #{ + default => 8, + desc => ?DESC("pool_size") + } + )}, + {timeout, + hoconsc:mk( + emqx_schema:duration_ms(), + #{ + default => <<"5s">>, + desc => ?DESC("timeout") + } + )}, + {wait_for_publish_confirmations, + hoconsc:mk( + boolean(), + #{ + default => true, + desc => ?DESC("wait_for_publish_confirmations") + } + )}, + {publish_confirmation_timeout, + hoconsc:mk( + emqx_schema:duration_ms(), + #{ + default => <<"30s">>, + desc => ?DESC("timeout") + } + )}, + + {virtual_host, + hoconsc:mk( + typerefl:binary(), + #{ + default => <<"/">>, + desc => ?DESC("virtual_host") + } + )}, + {heartbeat, + hoconsc:mk( + emqx_schema:duration_ms(), + #{ + default => <<"30s">>, + desc => ?DESC("heartbeat") + } + )}, + {auto_reconnect, + hoconsc:mk( + emqx_schema:duration_ms(), + #{ + default => <<"2s">>, + desc => ?DESC("auto_reconnect") + } + )}, + %% Things related to sending messages to RabbitMQ + {exchange, + hoconsc:mk( + typerefl:binary(), + #{ + required => true, + desc => ?DESC("exchange") + } + )}, + {routing_key, + hoconsc:mk( + typerefl:binary(), + #{ + required => true, + desc => ?DESC("routing_key") + } + )}, + {delivery_mode, + hoconsc:mk( + hoconsc:enum([non_persistent, persistent]), + #{ + default => non_persistent, + desc => ?DESC("delivery_mode") + } + )}, + {payload_template, + hoconsc:mk( + binary(), + #{ + default => <<"${.}">>, + desc => ?DESC("payload_template") + } + )} + ]. + +values(post) -> + maps:merge(values(put), #{name => <<"connector">>}); +values(get) -> + values(post); +values(put) -> + #{ + server => <<"localhost">>, + port => 5672, + enable => true, + pool_size => 8, + type => rabbitmq, + username => <<"guest">>, + password => <<"******">>, + routing_key => <<"my_routing_key">>, + payload_template => <<"">> + }; +values(_) -> + #{}. + +%% =================================================================== +%% Callbacks defined in emqx_resource +%% =================================================================== + +%% emqx_resource callback + +callback_mode() -> always_sync. + +%% emqx_resource callback + +-spec is_buffer_supported() -> boolean(). +is_buffer_supported() -> + %% We want to make use of EMQX's buffer mechanism + false. + +%% emqx_resource callback called when the resource is started + +-spec on_start(resource_id(), term()) -> {ok, resource_state()} | {error, _}. +on_start( + InstanceID, + #{ + pool_size := PoolSize, + payload_template := PayloadTemplate, + password := Password, + delivery_mode := InitialDeliveryMode + } = InitialConfig +) -> + DeliveryMode = + case InitialDeliveryMode of + non_persistent -> 1; + persistent -> 2 + end, + Config = InitialConfig#{ + password => emqx_secret:wrap(Password), + delivery_mode => DeliveryMode + }, + ?SLOG(info, #{ + msg => "starting_rabbitmq_connector", + connector => InstanceID, + config => emqx_utils:redact(Config) + }), + Options = [ + {config, Config}, + %% The pool_size is read by ecpool and decides the number of workers in + %% the pool + {pool_size, PoolSize}, + {pool, InstanceID} + ], + ProcessedTemplate = emqx_plugin_libs_rule:preproc_tmpl(PayloadTemplate), + State = #{ + poolname => InstanceID, + processed_payload_template => ProcessedTemplate, + config => Config + }, + case emqx_resource_pool:start(InstanceID, ?MODULE, Options) of + ok -> + {ok, State}; + {error, Reason} -> + LogMessage = + #{ + msg => "rabbitmq_connector_start_failed", + error_reason => Reason, + config => emqx_utils:redact(Config) + }, + ?SLOG(info, LogMessage), + {error, Reason} + end. + +%% emqx_resource callback called when the resource is stopped + +-spec on_stop(resource_id(), resource_state()) -> term(). +on_stop( + ResourceID, + #{poolname := PoolName} = _State +) -> + ?SLOG(info, #{ + msg => "stopping RabbitMQ connector", + connector => ResourceID + }), + Workers = [Worker || {_WorkerName, Worker} <- ecpool:workers(PoolName)], + Clients = [ + begin + {ok, Client} = ecpool_worker:client(Worker), + Client + end + || Worker <- Workers + ], + %% We need to stop the pool before stopping the workers as the pool monitors the workers + StopResult = emqx_resource_pool:stop(PoolName), + lists:foreach(fun stop_worker/1, Clients), + StopResult. + +stop_worker({Channel, Connection}) -> + amqp_channel:close(Channel), + amqp_connection:close(Connection). + +%% This is the callback function that is called by ecpool when the pool is +%% started + +-spec connect(term()) -> {ok, {pid(), pid()}, map()} | {error, term()}. +connect(Options) -> + Config = proplists:get_value(config, Options), + try + create_rabbitmq_connection_and_channel(Config) + catch + _:{error, Reason} -> + ?SLOG(error, #{ + msg => "rabbitmq_connector_connection_failed", + error_type => error, + error_reason => Reason, + config => emqx_utils:redact(Config) + }), + {error, Reason}; + Type:Reason -> + ?SLOG(error, #{ + msg => "rabbitmq_connector_connection_failed", + error_type => Type, + error_reason => Reason, + config => emqx_utils:redact(Config) + }), + {error, Reason} + end. + +create_rabbitmq_connection_and_channel(Config) -> + #{ + server := Host, + port := Port, + username := Username, + password := WrappedPassword, + timeout := Timeout, + virtual_host := VirtualHost, + heartbeat := Heartbeat, + wait_for_publish_confirmations := WaitForPublishConfirmations + } = Config, + Password = emqx_secret:unwrap(WrappedPassword), + RabbitMQConnectionOptions = + #amqp_params_network{ + host = erlang:binary_to_list(Host), + port = Port, + username = Username, + password = Password, + connection_timeout = Timeout, + virtual_host = VirtualHost, + heartbeat = Heartbeat + }, + {ok, RabbitMQConnection} = + case amqp_connection:start(RabbitMQConnectionOptions) of + {ok, Connection} -> + {ok, Connection}; + {error, Reason} -> + erlang:error({error, Reason}) + end, + {ok, RabbitMQChannel} = + case amqp_connection:open_channel(RabbitMQConnection) of + {ok, Channel} -> + {ok, Channel}; + {error, OpenChannelErrorReason} -> + erlang:error({error, OpenChannelErrorReason}) + end, + %% We need to enable confirmations if we want to wait for them + case WaitForPublishConfirmations of + true -> + case amqp_channel:call(RabbitMQChannel, #'confirm.select'{}) of + #'confirm.select_ok'{} -> + ok; + Error -> + ConfirmModeErrorReason = + erlang:iolist_to_binary( + io_lib:format( + "Could not enable RabbitMQ confirmation mode ~p", + [Error] + ) + ), + erlang:error({error, ConfirmModeErrorReason}) + end; + false -> + ok + end, + {ok, {RabbitMQConnection, RabbitMQChannel}, #{ + supervisees => [RabbitMQConnection, RabbitMQChannel] + }}. + +%% emqx_resource callback called to check the status of the resource + +-spec on_get_status(resource_id(), term()) -> + {connected, resource_state()} | {disconnected, resource_state(), binary()}. +on_get_status( + _InstId, + #{ + poolname := PoolName + } = State +) -> + Workers = [Worker || {_WorkerName, Worker} <- ecpool:workers(PoolName)], + Clients = [ + begin + {ok, Client} = ecpool_worker:client(Worker), + Client + end + || Worker <- Workers + ], + CheckResults = [ + check_worker(Client) + || Client <- Clients + ], + Connected = length(CheckResults) > 0 andalso lists:all(fun(R) -> R end, CheckResults), + case Connected of + true -> + {connected, State}; + false -> + {disconnected, State, <<"not_connected">>} + end; +on_get_status( + _InstId, + State +) -> + {disconnect, State, <<"not_connected: no connection pool in state">>}. + +check_worker({Channel, Connection}) -> + erlang:is_process_alive(Channel) andalso erlang:is_process_alive(Connection). + +%% emqx_resource callback that is called when a non-batch query is received + +-spec on_query(resource_id(), Request, resource_state()) -> query_result() when + Request :: {RequestType, Data}, + RequestType :: send_message, + Data :: map(). +on_query( + ResourceID, + {RequestType, Data}, + #{ + poolname := PoolName, + processed_payload_template := PayloadTemplate, + config := Config + } = State +) -> + ?SLOG(debug, #{ + msg => "RabbitMQ connector received query", + connector => ResourceID, + type => RequestType, + data => Data, + state => emqx_utils:redact(State) + }), + MessageData = format_data(PayloadTemplate, Data), + ecpool:pick_and_do( + PoolName, + {?MODULE, publish_messages, [Config, [MessageData]]}, + no_handover + ). + +%% emqx_resource callback that is called when a batch query is received + +-spec on_batch_query(resource_id(), BatchReq, resource_state()) -> query_result() when + BatchReq :: nonempty_list({'send_message', map()}). +on_batch_query( + ResourceID, + BatchReq, + State +) -> + ?SLOG(debug, #{ + msg => "RabbitMQ connector received batch query", + connector => ResourceID, + data => BatchReq, + state => emqx_utils:redact(State) + }), + %% Currently we only support batch requests with the send_message key + {Keys, MessagesToInsert} = lists:unzip(BatchReq), + ensure_keys_are_of_type_send_message(Keys), + %% Pick out the payload template + #{ + processed_payload_template := PayloadTemplate, + poolname := PoolName, + config := Config + } = State, + %% Create batch payload + FormattedMessages = [ + format_data(PayloadTemplate, Data) + || Data <- MessagesToInsert + ], + %% Publish the messages + ecpool:pick_and_do( + PoolName, + {?MODULE, publish_messages, [Config, FormattedMessages]}, + no_handover + ). + +publish_messages( + {_Connection, Channel}, + #{ + delivery_mode := DeliveryMode, + routing_key := RoutingKey, + exchange := Exchange, + wait_for_publish_confirmations := WaitForPublishConfirmations, + publish_confirmation_timeout := PublishConfirmationTimeout + } = _Config, + Messages +) -> + MessageProperties = #'P_basic'{ + headers = [], + delivery_mode = DeliveryMode + }, + Method = #'basic.publish'{ + exchange = Exchange, + routing_key = RoutingKey + }, + _ = [ + amqp_channel:cast( + Channel, + Method, + #amqp_msg{ + payload = Message, + props = MessageProperties + } + ) + || Message <- Messages + ], + case WaitForPublishConfirmations of + true -> + case amqp_channel:wait_for_confirms(Channel, PublishConfirmationTimeout) of + true -> + ok; + false -> + erlang:error( + {recoverable_error, + <<"RabbitMQ: Got NACK when waiting for message acknowledgment.">>} + ); + timeout -> + erlang:error( + {recoverable_error, + <<"RabbitMQ: Timeout when waiting for message acknowledgment.">>} + ) + end; + false -> + ok + end. + +ensure_keys_are_of_type_send_message(Keys) -> + case lists:all(fun is_send_message_atom/1, Keys) of + true -> + ok; + false -> + erlang:error( + {unrecoverable_error, + <<"Unexpected type for batch message (Expected send_message)">>} + ) + end. + +is_send_message_atom(send_message) -> + true; +is_send_message_atom(_) -> + false. + +format_data([], Msg) -> + emqx_utils_json:encode(Msg); +format_data(Tokens, Msg) -> + emqx_plugin_libs_rule:proc_tmpl(Tokens, Msg). diff --git a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_SUITE.erl b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_SUITE.erl new file mode 100644 index 000000000..45a8693e6 --- /dev/null +++ b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_SUITE.erl @@ -0,0 +1,371 @@ +%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_rabbitmq_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("emqx_connector/include/emqx_connector.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +%% See comment in +%% lib-ee/emqx_ee_connector/test/ee_connector_rabbitmq_SUITE.erl for how to +%% run this without bringing up the whole CI infrastucture + +rabbit_mq_host() -> + <<"rabbitmq">>. + +rabbit_mq_port() -> + 5672. + +rabbit_mq_exchange() -> + <<"messages">>. + +rabbit_mq_queue() -> + <<"test_queue">>. + +rabbit_mq_routing_key() -> + <<"test_routing_key">>. + +get_channel_connection(Config) -> + proplists:get_value(channel_connection, Config). + +%%------------------------------------------------------------------------------ +%% Common Test Setup, Teardown and Testcase List +%%------------------------------------------------------------------------------ + +init_per_suite(Config) -> + % snabbkaffe:fix_ct_logging(), + case + emqx_common_test_helpers:is_tcp_server_available( + erlang:binary_to_list(rabbit_mq_host()), rabbit_mq_port() + ) + of + true -> + emqx_common_test_helpers:render_and_load_app_config(emqx_conf), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + ok = emqx_connector_test_helpers:start_apps([emqx_resource]), + {ok, _} = application:ensure_all_started(emqx_connector), + {ok, _} = application:ensure_all_started(emqx_ee_connector), + {ok, _} = application:ensure_all_started(emqx_ee_bridge), + {ok, _} = application:ensure_all_started(amqp_client), + emqx_mgmt_api_test_util:init_suite(), + ChannelConnection = setup_rabbit_mq_exchange_and_queue(), + [{channel_connection, ChannelConnection} | Config]; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_rabbitmq); + _ -> + {skip, no_rabbitmq} + end + end. + +setup_rabbit_mq_exchange_and_queue() -> + %% Create an exachange and a queue + {ok, Connection} = + amqp_connection:start(#amqp_params_network{ + host = erlang:binary_to_list(rabbit_mq_host()), + port = rabbit_mq_port() + }), + {ok, Channel} = amqp_connection:open_channel(Connection), + %% Create an exchange + #'exchange.declare_ok'{} = + amqp_channel:call( + Channel, + #'exchange.declare'{ + exchange = rabbit_mq_exchange(), + type = <<"topic">> + } + ), + %% Create a queue + #'queue.declare_ok'{} = + amqp_channel:call( + Channel, + #'queue.declare'{queue = rabbit_mq_queue()} + ), + %% Bind the queue to the exchange + #'queue.bind_ok'{} = + amqp_channel:call( + Channel, + #'queue.bind'{ + queue = rabbit_mq_queue(), + exchange = rabbit_mq_exchange(), + routing_key = rabbit_mq_routing_key() + } + ), + #{ + connection => Connection, + channel => Channel + }. + +end_per_suite(Config) -> + #{ + connection := Connection, + channel := Channel + } = get_channel_connection(Config), + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([emqx_conf]), + ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), + _ = application:stop(emqx_connector), + _ = application:stop(emqx_ee_connector), + _ = application:stop(emqx_bridge), + %% Close the channel + ok = amqp_channel:close(Channel), + %% Close the connection + ok = amqp_connection:close(Connection). + +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(_, _Config) -> + ok. + +all() -> + emqx_common_test_helpers:all(?MODULE). + +rabbitmq_config(Config) -> + %%SQL = maps:get(sql, Config, sql_insert_template_for_bridge()), + BatchSize = maps:get(batch_size, Config, 1), + BatchTime = maps:get(batch_time_ms, Config, 0), + Name = atom_to_binary(?MODULE), + Server = maps:get(server, Config, rabbit_mq_host()), + Port = maps:get(port, Config, rabbit_mq_port()), + Template = maps:get(payload_template, Config, <<"">>), + ConfigString = + io_lib:format( + "bridges.rabbitmq.~s {\n" + " enable = true\n" + " server = \"~s\"\n" + " port = ~p\n" + " username = \"guest\"\n" + " password = \"guest\"\n" + " routing_key = \"~s\"\n" + " exchange = \"~s\"\n" + " payload_template = \"~s\"\n" + " resource_opts = {\n" + " batch_size = ~b\n" + " batch_time = ~bms\n" + " }\n" + "}\n", + [ + Name, + Server, + Port, + rabbit_mq_routing_key(), + rabbit_mq_exchange(), + Template, + BatchSize, + BatchTime + ] + ), + ct:pal(ConfigString), + parse_and_check(ConfigString, <<"rabbitmq">>, Name). + +parse_and_check(ConfigString, BridgeType, Name) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{BridgeType := #{Name := RetConfig}}} = RawConf, + RetConfig. + +make_bridge(Config) -> + Type = <<"rabbitmq">>, + Name = atom_to_binary(?MODULE), + BridgeConfig = rabbitmq_config(Config), + {ok, _} = emqx_bridge:create( + Type, + Name, + BridgeConfig + ), + emqx_bridge_resource:bridge_id(Type, Name). + +delete_bridge() -> + Type = <<"rabbitmq">>, + Name = atom_to_binary(?MODULE), + {ok, _} = emqx_bridge:remove(Type, Name), + ok. + +%%------------------------------------------------------------------------------ +%% Test Cases +%%------------------------------------------------------------------------------ + +t_make_delete_bridge(_Config) -> + make_bridge(#{}), + %% Check that the new brige is in the list of bridges + Bridges = emqx_bridge:list(), + Name = atom_to_binary(?MODULE), + IsRightName = + fun + (#{name := BName}) when BName =:= Name -> + true; + (_) -> + false + end, + ?assert(lists:any(IsRightName, Bridges)), + delete_bridge(), + BridgesAfterDelete = emqx_bridge:list(), + ?assertNot(lists:any(IsRightName, BridgesAfterDelete)), + ok. + +t_make_delete_bridge_non_existing_server(_Config) -> + make_bridge(#{server => <<"non_existing_server">>, port => 3174}), + %% Check that the new brige is in the list of bridges + Bridges = emqx_bridge:list(), + Name = atom_to_binary(?MODULE), + IsRightName = + fun + (#{name := BName}) when BName =:= Name -> + true; + (_) -> + false + end, + ?assert(lists:any(IsRightName, Bridges)), + delete_bridge(), + BridgesAfterDelete = emqx_bridge:list(), + ?assertNot(lists:any(IsRightName, BridgesAfterDelete)), + ok. + +t_send_message_query(Config) -> + BridgeID = make_bridge(#{batch_size => 1}), + Payload = #{<<"key">> => 42, <<"data">> => <<"RabbitMQ">>, <<"timestamp">> => 10000}, + %% This will use the SQL template included in the bridge + emqx_bridge:send_message(BridgeID, Payload), + %% Check that the data got to the database + ?assertEqual(Payload, receive_simple_test_message(Config)), + delete_bridge(), + ok. + +t_send_message_query_with_template(Config) -> + BridgeID = make_bridge(#{ + batch_size => 1, + payload_template => + << + "{" + " \\\"key\\\": ${key}," + " \\\"data\\\": \\\"${data}\\\"," + " \\\"timestamp\\\": ${timestamp}," + " \\\"secret\\\": 42" + "}" + >> + }), + Payload = #{ + <<"key">> => 7, + <<"data">> => <<"RabbitMQ">>, + <<"timestamp">> => 10000 + }, + emqx_bridge:send_message(BridgeID, Payload), + %% Check that the data got to the database + ExpectedResult = Payload#{ + <<"secret">> => 42 + }, + ?assertEqual(ExpectedResult, receive_simple_test_message(Config)), + delete_bridge(), + ok. + +t_send_simple_batch(Config) -> + BridgeConf = + #{ + batch_size => 100 + }, + BridgeID = make_bridge(BridgeConf), + Payload = #{<<"key">> => 42, <<"data">> => <<"RabbitMQ">>, <<"timestamp">> => 10000}, + emqx_bridge:send_message(BridgeID, Payload), + ?assertEqual(Payload, receive_simple_test_message(Config)), + delete_bridge(), + ok. + +t_send_simple_batch_with_template(Config) -> + BridgeConf = + #{ + batch_size => 100, + payload_template => + << + "{" + " \\\"key\\\": ${key}," + " \\\"data\\\": \\\"${data}\\\"," + " \\\"timestamp\\\": ${timestamp}," + " \\\"secret\\\": 42" + "}" + >> + }, + BridgeID = make_bridge(BridgeConf), + Payload = #{ + <<"key">> => 7, + <<"data">> => <<"RabbitMQ">>, + <<"timestamp">> => 10000 + }, + emqx_bridge:send_message(BridgeID, Payload), + ExpectedResult = Payload#{ + <<"secret">> => 42 + }, + ?assertEqual(ExpectedResult, receive_simple_test_message(Config)), + delete_bridge(), + ok. + +t_heavy_batching(Config) -> + NumberOfMessages = 20000, + BridgeConf = #{ + batch_size => 10173, + batch_time_ms => 50 + }, + BridgeID = make_bridge(BridgeConf), + SendMessage = fun(Key) -> + Payload = #{ + <<"key">> => Key + }, + emqx_bridge:send_message(BridgeID, Payload) + end, + [SendMessage(Key) || Key <- lists:seq(1, NumberOfMessages)], + AllMessages = lists:foldl( + fun(_, Acc) -> + Message = receive_simple_test_message(Config), + #{<<"key">> := Key} = Message, + Acc#{Key => true} + end, + #{}, + lists:seq(1, NumberOfMessages) + ), + ?assertEqual(NumberOfMessages, maps:size(AllMessages)), + delete_bridge(), + ok. + +receive_simple_test_message(Config) -> + #{channel := Channel} = get_channel_connection(Config), + #'basic.consume_ok'{consumer_tag = ConsumerTag} = + amqp_channel:call( + Channel, + #'basic.consume'{ + queue = rabbit_mq_queue() + } + ), + receive + %% This is the first message received + #'basic.consume_ok'{} -> + ok + end, + receive + {#'basic.deliver'{delivery_tag = DeliveryTag}, Content} -> + %% Ack the message + amqp_channel:cast(Channel, #'basic.ack'{delivery_tag = DeliveryTag}), + %% Cancel the consumer + #'basic.cancel_ok'{consumer_tag = ConsumerTag} = + amqp_channel:call(Channel, #'basic.cancel'{consumer_tag = ConsumerTag}), + emqx_utils_json:decode(Content#amqp_msg.payload) + end. + +rabbitmq_config() -> + Config = + #{ + server => rabbit_mq_host(), + port => 5672, + exchange => rabbit_mq_exchange(), + routing_key => rabbit_mq_routing_key() + }, + #{<<"config">> => Config}. + +test_data() -> + #{<<"msg_field">> => <<"Hello">>}. diff --git a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl new file mode 100644 index 000000000..6b6ad617f --- /dev/null +++ b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl @@ -0,0 +1,232 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_rabbitmq_connector_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_connector.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +%% This test SUITE requires a running RabbitMQ instance. If you don't want to +%% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script +%% you can create a clickhouse instance with the following command. +%% 5672 is the default port for AMQP 0-9-1 and 15672 is the default port for +%% the HTTP managament interface. +%% +%% docker run -it --rm --name rabbitmq -p 127.0.0.1:5672:5672 -p 127.0.0.1:15672:15672 rabbitmq:3.11-management + +rabbit_mq_host() -> + <<"rabbitmq">>. + +rabbit_mq_port() -> + 5672. + +rabbit_mq_exchange() -> + <<"test_exchange">>. + +rabbit_mq_queue() -> + <<"test_queue">>. + +rabbit_mq_routing_key() -> + <<"test_routing_key">>. + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + case + emqx_common_test_helpers:is_tcp_server_available( + erlang:binary_to_list(rabbit_mq_host()), rabbit_mq_port() + ) + of + true -> + ok = emqx_common_test_helpers:start_apps([emqx_conf]), + ok = emqx_connector_test_helpers:start_apps([emqx_resource]), + {ok, _} = application:ensure_all_started(emqx_connector), + {ok, _} = application:ensure_all_started(emqx_ee_connector), + {ok, _} = application:ensure_all_started(amqp_client), + ChannelConnection = setup_rabbit_mq_exchange_and_queue(), + [{channel_connection, ChannelConnection} | Config]; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_rabbitmq); + _ -> + {skip, no_rabbitmq} + end + end. + +setup_rabbit_mq_exchange_and_queue() -> + %% Create an exachange and a queue + {ok, Connection} = + amqp_connection:start(#amqp_params_network{ + host = erlang:binary_to_list(rabbit_mq_host()), + port = rabbit_mq_port() + }), + {ok, Channel} = amqp_connection:open_channel(Connection), + %% Create an exchange + #'exchange.declare_ok'{} = + amqp_channel:call( + Channel, + #'exchange.declare'{ + exchange = rabbit_mq_exchange(), + type = <<"topic">> + } + ), + %% Create a queue + #'queue.declare_ok'{} = + amqp_channel:call( + Channel, + #'queue.declare'{queue = rabbit_mq_queue()} + ), + %% Bind the queue to the exchange + #'queue.bind_ok'{} = + amqp_channel:call( + Channel, + #'queue.bind'{ + queue = rabbit_mq_queue(), + exchange = rabbit_mq_exchange(), + routing_key = rabbit_mq_routing_key() + } + ), + #{ + connection => Connection, + channel => Channel + }. + +get_channel_connection(Config) -> + proplists:get_value(channel_connection, Config). + +end_per_suite(Config) -> + #{ + connection := Connection, + channel := Channel + } = get_channel_connection(Config), + ok = emqx_common_test_helpers:stop_apps([emqx_conf]), + ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), + _ = application:stop(emqx_connector), + %% Close the channel + ok = amqp_channel:close(Channel), + %% Close the connection + ok = amqp_connection:close(Connection). + +% %%------------------------------------------------------------------------------ +% %% Testcases +% %%------------------------------------------------------------------------------ + +t_lifecycle(Config) -> + perform_lifecycle_check( + erlang:atom_to_binary(?MODULE), + rabbitmq_config(), + Config + ). + +perform_lifecycle_check(ResourceID, InitialConfig, TestConfig) -> + #{ + channel := Channel + } = get_channel_connection(TestConfig), + {ok, #{config := CheckedConfig}} = + emqx_resource:check_config(emqx_bridge_rabbitmq_connector, InitialConfig), + {ok, #{ + state := #{poolname := PoolName} = State, + status := InitialStatus + }} = + emqx_resource:create_local( + ResourceID, + ?CONNECTOR_RESOURCE_GROUP, + emqx_bridge_rabbitmq_connector, + CheckedConfig, + #{} + ), + ?assertEqual(InitialStatus, connected), + %% Instance should match the state and status of the just started resource + {ok, ?CONNECTOR_RESOURCE_GROUP, #{ + state := State, + status := InitialStatus + }} = + emqx_resource:get_instance(ResourceID), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceID)), + %% Perform query as further check that the resource is working as expected + perform_query(ResourceID, Channel), + ?assertEqual(ok, emqx_resource:stop(ResourceID)), + %% Resource will be listed still, but state will be changed and healthcheck will fail + %% as the worker no longer exists. + {ok, ?CONNECTOR_RESOURCE_GROUP, #{ + state := State, + status := StoppedStatus + }} = emqx_resource:get_instance(ResourceID), + ?assertEqual(stopped, StoppedStatus), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(ResourceID)), + % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), + % Can call stop/1 again on an already stopped instance + ?assertEqual(ok, emqx_resource:stop(ResourceID)), + % Make sure it can be restarted and the healthchecks and queries work properly + ?assertEqual(ok, emqx_resource:restart(ResourceID)), + % async restart, need to wait resource + timer:sleep(500), + {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = + emqx_resource:get_instance(ResourceID), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceID)), + %% Check that everything is working again by performing a query + perform_query(ResourceID, Channel), + % Stop and remove the resource in one go. + ?assertEqual(ok, emqx_resource:remove_local(ResourceID)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), + % Should not even be able to get the resource data out of ets now unlike just stopping. + ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceID)). + +% %%------------------------------------------------------------------------------ +% %% Helpers +% %%------------------------------------------------------------------------------ + +perform_query(PoolName, Channel) -> + %% Send message to queue: + ok = emqx_resource:query(PoolName, {query, test_data()}), + %% Get the message from queue: + ok = receive_simple_test_message(Channel). + +receive_simple_test_message(Channel) -> + #'basic.consume_ok'{consumer_tag = ConsumerTag} = + amqp_channel:call( + Channel, + #'basic.consume'{ + queue = rabbit_mq_queue() + } + ), + receive + %% This is the first message received + #'basic.consume_ok'{} -> + ok + end, + receive + {#'basic.deliver'{delivery_tag = DeliveryTag}, Content} -> + Expected = test_data(), + ?assertEqual(Expected, emqx_utils_json:decode(Content#amqp_msg.payload)), + %% Ack the message + amqp_channel:cast(Channel, #'basic.ack'{delivery_tag = DeliveryTag}), + %% Cancel the consumer + #'basic.cancel_ok'{consumer_tag = ConsumerTag} = + amqp_channel:call(Channel, #'basic.cancel'{consumer_tag = ConsumerTag}), + ok + end. + +rabbitmq_config() -> + Config = + #{ + server => rabbit_mq_host(), + port => 5672, + username => <<"guest">>, + password => <<"guest">>, + exchange => rabbit_mq_exchange(), + routing_key => rabbit_mq_routing_key() + }, + #{<<"config">> => Config}. + +test_data() -> + #{<<"msg_field">> => <<"Hello">>}. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 528fcd972..0344c84c4 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -846,6 +846,8 @@ typename_to_spec("bucket_name()", _Mod) -> #{type => string, example => <<"retainer">>}; typename_to_spec("json_binary()", _Mod) -> #{type => string, example => <<"{\"a\": [1,true]}">>}; +typename_to_spec("port_number()", _Mod) -> + range("1..65535"); typename_to_spec(Name, Mod) -> Spec = range(Name), Spec1 = remote_module_type(Spec, Name, Mod), diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 7c48e8ee4..80f270b13 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -134,6 +134,9 @@ %% when calling emqx_resource:stop/1 -callback on_stop(resource_id(), resource_state()) -> term(). +%% when calling emqx_resource:get_callback_mode/1 +-callback callback_mode() -> callback_mode(). + %% when calling emqx_resource:query/3 -callback on_query(resource_id(), Request :: term(), resource_state()) -> query_result(). diff --git a/changes/ee/feat-10534.md b/changes/ee/feat-10534.md new file mode 100644 index 000000000..e87167d6a --- /dev/null +++ b/changes/ee/feat-10534.md @@ -0,0 +1 @@ +A RabbitMQ bridge has been added. This bridge makes it possible to forward messages from EMQX to RabbitMQ. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 39f6f2043..decf7d033 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -13,7 +13,8 @@ emqx_bridge_opents, emqx_bridge_pulsar, emqx_bridge_sqlserver, - emqx_bridge_rocketmq + emqx_bridge_rocketmq, + emqx_bridge_rabbitmq ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 121ba5c0e..80c627cc8 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -39,7 +39,8 @@ api_schemas(Method) -> ref(emqx_bridge_opents, Method), ref(emqx_bridge_pulsar, Method ++ "_producer"), ref(emqx_bridge_oracle, Method), - ref(emqx_bridge_iotdb, Method) + ref(emqx_bridge_iotdb, Method), + ref(emqx_bridge_rabbitmq, Method) ]. schema_modules() -> @@ -63,7 +64,8 @@ schema_modules() -> emqx_bridge_opents, emqx_bridge_pulsar, emqx_bridge_oracle, - emqx_bridge_iotdb + emqx_bridge_iotdb, + emqx_bridge_rabbitmq ]. examples(Method) -> @@ -106,7 +108,8 @@ resource_type(sqlserver) -> emqx_bridge_sqlserver_connector; resource_type(opents) -> emqx_bridge_opents_connector; resource_type(pulsar_producer) -> emqx_bridge_pulsar_impl_producer; resource_type(oracle) -> emqx_oracle; -resource_type(iotdb) -> emqx_bridge_iotdb_impl. +resource_type(iotdb) -> emqx_bridge_iotdb_impl; +resource_type(rabbitmq) -> emqx_bridge_rabbitmq_connector. fields(bridges) -> [ @@ -192,7 +195,7 @@ fields(bridges) -> )} ] ++ kafka_structs() ++ pulsar_structs() ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++ - pgsql_structs() ++ clickhouse_structs() ++ sqlserver_structs(). + pgsql_structs() ++ clickhouse_structs() ++ sqlserver_structs() ++ rabbitmq_structs(). mongodb_structs() -> [ @@ -323,3 +326,15 @@ kafka_producer_converter(Map, Opts) -> end, Map ). + +rabbitmq_structs() -> + [ + {rabbitmq, + mk( + hoconsc:map(name, ref(emqx_bridge_rabbitmq, "config")), + #{ + desc => <<"RabbitMQ Bridge Config">>, + required => false + } + )} + ]. diff --git a/mix.exs b/mix.exs index 9383fa085..564d81ccf 100644 --- a/mix.exs +++ b/mix.exs @@ -174,7 +174,8 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_sqlserver, :emqx_bridge_pulsar, :emqx_oracle, - :emqx_bridge_oracle + :emqx_bridge_oracle, + :emqx_bridge_rabbitmq ]) end @@ -189,7 +190,22 @@ defmodule EMQXUmbrella.MixProject do {:snappyer, "1.2.8", override: true}, {:crc32cer, "0.1.8", override: true}, {:supervisor3, "1.1.12", override: true}, - {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true} + {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true}, + # The following two are dependencies of rabbit_common. They are needed here to + # make mix not complain about conflicting versions + {:thoas, github: "emqx/thoas", tag: "v1.0.0", override: true}, + {:credentials_obfuscation, + github: "emqx/credentials-obfuscation", tag: "v3.2.0", override: true}, + {:rabbit_common, + github: "emqx/rabbitmq-server", + tag: "v3.11.13-emqx", + sparse: "deps/rabbit_common", + override: true}, + {:amqp_client, + github: "emqx/rabbitmq-server", + tag: "v3.11.13-emqx", + sparse: "deps/amqp_client", + override: true} ] end @@ -321,7 +337,7 @@ defmodule EMQXUmbrella.MixProject do emqx_plugin_libs: :load, esasl: :load, observer_cli: :permanent, - tools: :load, + tools: :permanent, covertool: :load, system_monitor: :load, emqx_utils: :load, @@ -385,6 +401,7 @@ defmodule EMQXUmbrella.MixProject do emqx_bridge_sqlserver: :permanent, emqx_oracle: :permanent, emqx_bridge_oracle: :permanent, + emqx_bridge_rabbitmq: :permanent, emqx_ee_schema_registry: :permanent ], else: [] diff --git a/rebar.config.erl b/rebar.config.erl index bb3bbbab6..a48a365c9 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -98,6 +98,7 @@ is_community_umbrella_app("apps/emqx_bridge_timescale") -> false; is_community_umbrella_app("apps/emqx_bridge_oracle") -> false; is_community_umbrella_app("apps/emqx_bridge_sqlserver") -> false; is_community_umbrella_app("apps/emqx_oracle") -> false; +is_community_umbrella_app("apps/emqx_bridge_rabbitmq") -> false; is_community_umbrella_app(_) -> true. is_jq_supported() -> @@ -404,7 +405,7 @@ relx_apps(ReleaseType, Edition) -> {emqx_plugin_libs, load}, {esasl, load}, observer_cli, - {tools, load}, + tools, {covertool, load}, % started by emqx_machine {system_monitor, load}, @@ -476,6 +477,7 @@ relx_apps_per_edition(ee) -> emqx_bridge_sqlserver, emqx_oracle, emqx_bridge_oracle, + emqx_bridge_rabbitmq, emqx_ee_schema_registry ]; relx_apps_per_edition(ce) -> diff --git a/rel/i18n/emqx_bridge_rabbitmq.hocon b/rel/i18n/emqx_bridge_rabbitmq.hocon new file mode 100644 index 000000000..a27dc4f37 --- /dev/null +++ b/rel/i18n/emqx_bridge_rabbitmq.hocon @@ -0,0 +1,34 @@ +emqx_bridge_rabbitmq { + + local_topic.desc: + """The MQTT topic filter to be forwarded to RabbitMQ. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded. + NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" + + local_topic.label: + """Local Topic""" + + config_enable.desc: + """Enable or disable this bridge""" + + config_enable.label: + """Enable or Disable Bridge""" + + desc_config.desc: + """Configuration for a RabbitMQ bridge.""" + + desc_config.label: + """RabbitMQ Bridge Configuration""" + + desc_type.desc: + """The Bridge Type""" + + desc_type.label: + """Bridge Type""" + + desc_name.desc: + """Bridge name.""" + + desc_name.label: + """Bridge Name""" + +} diff --git a/rel/i18n/emqx_bridge_rabbitmq_connector.hocon b/rel/i18n/emqx_bridge_rabbitmq_connector.hocon new file mode 100644 index 000000000..a0f6161d4 --- /dev/null +++ b/rel/i18n/emqx_bridge_rabbitmq_connector.hocon @@ -0,0 +1,100 @@ + +emqx_bridge_rabbitmq_connector { + +server.desc: +"""The RabbitMQ server address that you want to connect to (for example, localhost).""" + +server.label: +"""Server""" + +port.desc: +"""The port number on which the RabbitMQ server is listening (default is 5672).""" + +port.label: +"""Port""" + +username.desc: +"""The username used to authenticate with the RabbitMQ server.""" + +username.label: +"""Username""" + +password.desc: +"""The password used to authenticate with the RabbitMQ server.""" + +password.label: +"""Password""" + +pool_size.desc: +"""The size of the connection pool.""" + +pool_size.label: +"""Pool Size""" + +timeout.desc: +"""The timeout for waiting on the connection to be established.""" + +timeout.label: +"""Connection Timeout""" + +virtual_host.desc: +"""The virtual host to use when connecting to the RabbitMQ server.""" + +virtual_host.label: +"""Virtual Host""" + +heartbeat.desc: +"""The interval for sending heartbeat messages to the RabbitMQ server.""" + +heartbeat.label: +"""Heartbeat""" + +auto_reconnect.desc: +"""The interval for attempting to reconnect to the RabbitMQ server if the connection is lost.""" + +auto_reconnect.label: +"""Auto Reconnect""" + +exchange.desc: +"""The name of the RabbitMQ exchange where the messages will be sent.""" + +exchange.label: +"""Exchange""" + +exchange_type.desc: +"""The type of the RabbitMQ exchange (direct, fanout, or topic).""" + +exchange_type.label: +"""Exchange Type""" + +routing_key.desc: +"""The routing key used to route messages to the correct queue in the RabbitMQ exchange.""" + +routing_key.label: +"""Routing Key""" + +delivery_mode.desc: +"""The delivery mode for messages published to RabbitMQ. Delivery mode non_persistent (1) is suitable for messages that don't require persistence across RabbitMQ restarts, whereas delivery mode persistent (2) is designed for messages that must survive RabbitMQ restarts.""" + +delivery_mode.label: +"""Message Delivery Mode""" + +payload_template.desc: +"""The template for formatting the payload of the message before sending it to RabbitMQ. Template placeholders, such as ${field1.sub_field}, will be substituted with the respective field's value. When left empty, the entire input message will be used as the payload, formatted as a JSON text. This behavior is equivalent to specifying ${.} as the payload template.""" + +payload_template.label: +"""Payload Template""" + +publish_confirmation_timeout.desc: +"""The timeout for waiting for RabbitMQ to confirm message publication when using publisher confirms.""" + +publish_confirmation_timeout.label: +"""Publish Confirmation Timeout""" + +wait_for_publish_confirmations.desc: +"""A boolean value that indicates whether to wait for RabbitMQ to confirm message publication when using publisher confirms.""" + +wait_for_publish_confirmations.label: +"""Wait for Publish Confirmations""" + +} diff --git a/scripts/check-elixir-deps-discrepancies.exs b/scripts/check-elixir-deps-discrepancies.exs index eee0a9e67..408079d7d 100755 --- a/scripts/check-elixir-deps-discrepancies.exs +++ b/scripts/check-elixir-deps-discrepancies.exs @@ -36,6 +36,9 @@ rebar_deps = {:git, _, {:ref, ref}} -> to_string(ref) + + {:git_subdir, _, {:ref, ref}, _} -> + to_string(ref) end {name, ref} diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 62f616576..4824fbdf3 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -200,6 +200,9 @@ for dep in ${CT_DEPS}; do iotdb) FILES+=( '.ci/docker-compose-file/docker-compose-iotdb.yaml' ) ;; + rabbitmq) + FILES+=( '.ci/docker-compose-file/docker-compose-rabbitmq.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 From 5643c12930d3b4608470af095660a81a838a39aa Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 8 May 2023 10:19:49 -0300 Subject: [PATCH 038/144] chore: bump gproc -> 0.9.0.1 Includes fix: https://github.com/uwiger/gproc/pull/193 Prior to the fix, when using the `random` pool strategy, one of the workers receives about double the load of other workers, which decreases throughput of bridges like webhook. --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 18119607e..425c49fb3 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -24,7 +24,7 @@ {deps, [ {emqx_utils, {path, "../emqx_utils"}}, {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}}, - {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}, + {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.1"}}}, diff --git a/mix.exs b/mix.exs index 469337af8..51b4cb2a8 100644 --- a/mix.exs +++ b/mix.exs @@ -50,7 +50,7 @@ defmodule EMQXUmbrella.MixProject do {:covertool, github: "zmstone/covertool", tag: "2.0.4.1", override: true}, {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true}, {:ehttpc, github: "emqx/ehttpc", tag: "0.4.8", override: true}, - {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, + {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.6", override: true}, diff --git a/rebar.config b/rebar.config index 03b03345d..c229480db 100644 --- a/rebar.config +++ b/rebar.config @@ -57,7 +57,7 @@ , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}} , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.8"}}} - , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} + , {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}} From b98a7168c40095aba65cf7140acd83555c649127 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 9 May 2023 17:18:23 +0300 Subject: [PATCH 039/144] fix(emqx_gateway): convert and clear authentication certificates --- apps/emqx/src/emqx_authentication_config.erl | 10 +++- apps/emqx_gateway/src/emqx_gateway_conf.erl | 60 ++++++++++++++++++-- changes/ce/fix-10653.en.md | 1 + 3 files changed, 62 insertions(+), 9 deletions(-) create mode 100644 changes/ce/fix-10653.en.md diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index be3b35f57..98c0a19f8 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -29,9 +29,13 @@ authn_type/1 ]). --ifdef(TEST). --export([convert_certs/2, convert_certs/3, clear_certs/2]). --endif. +%% Used in emqx_gateway +-export([ + certs_dir/2, + convert_certs/2, + convert_certs/3, + clear_certs/2 +]). -export_type([config/0]). diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index 56a3e2068..da86d6a58 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -448,10 +448,12 @@ pre_config_update(_, {add_authn, GwName, Conf}, RawConf) -> ) of undefined -> + CertsDir = authn_certs_dir(GwName, Conf), + Conf1 = emqx_authentication_config:convert_certs(CertsDir, Conf), {ok, emqx_utils_maps:deep_merge( RawConf, - #{GwName => #{?AUTHN_BIN => Conf}} + #{GwName => #{?AUTHN_BIN => Conf1}} )}; _ -> badres_authn(already_exist, GwName) @@ -469,7 +471,9 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> Listener -> case maps:get(?AUTHN_BIN, Listener, undefined) of undefined -> - NListener = maps:put(?AUTHN_BIN, Conf, Listener), + CertsDir = authn_certs_dir(GwName, LType, LName, Conf), + Conf1 = emqx_authentication_config:convert_certs(CertsDir, Conf), + NListener = maps:put(?AUTHN_BIN, Conf1, Listener), NGateway = #{ GwName => #{ @@ -490,8 +494,10 @@ pre_config_update(_, {update_authn, GwName, Conf}, RawConf) -> of undefined -> badres_authn(not_found, GwName); - _Authn -> - {ok, emqx_utils_maps:deep_put([GwName, ?AUTHN_BIN], RawConf, Conf)} + OldAuthnConf -> + CertsDir = authn_certs_dir(GwName, Conf), + Conf1 = emqx_authentication_config:convert_certs(CertsDir, Conf, OldAuthnConf), + {ok, emqx_utils_maps:deep_put([GwName, ?AUTHN_BIN], RawConf, Conf1)} end; pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> case @@ -507,10 +513,16 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> case maps:get(?AUTHN_BIN, Listener, undefined) of undefined -> badres_listener_authn(not_found, GwName, LType, LName); - _Auth -> + OldAuthnConf -> + CertsDir = authn_certs_dir(GwName, LType, LName, OldAuthnConf), + Conf1 = emqx_authentication_config:convert_certs( + CertsDir, + Conf, + OldAuthnConf + ), NListener = maps:put( ?AUTHN_BIN, - Conf, + Conf1, Listener ), {ok, @@ -522,12 +534,36 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> end end; pre_config_update(_, {remove_authn, GwName}, RawConf) -> + case + emqx_utils_maps:deep_get( + [GwName, ?AUTHN_BIN], RawConf, undefined + ) + of + undefined -> + ok; + OldAuthnConf -> + CertsDir = authn_certs_dir(GwName, OldAuthnConf), + emqx_authentication_config:clear_certs(CertsDir, OldAuthnConf) + end, {ok, emqx_utils_maps:deep_remove( [GwName, ?AUTHN_BIN], RawConf )}; pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) -> Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN], + case + emqx_utils_maps:deep_get( + Path, + RawConf, + undefined + ) + of + undefined -> + ok; + OldAuthnConf -> + CertsDir = authn_certs_dir(GwName, LType, LName, OldAuthnConf), + emqx_authentication_config:clear_certs(CertsDir, OldAuthnConf) + end, {ok, emqx_utils_maps:deep_remove(Path, RawConf)}; pre_config_update(_, UnknownReq, _RawConf) -> logger:error("Unknown configuration update request: ~0p", [UnknownReq]), @@ -678,6 +714,18 @@ apply_to_gateway_basic_confs(_Fun, _GwName, Conf) -> certs_dir(GwName) when is_binary(GwName) -> GwName. +authn_certs_dir(GwName, ListenerType, ListenerName, AuthnConf) -> + ChainName = emqx_gateway_utils:listener_chain(GwName, ListenerType, ListenerName), + emqx_authentication_config:certs_dir(ChainName, AuthnConf). + +authn_certs_dir(GwName, AuthnConf) when is_binary(GwName) -> + authn_certs_dir(binary_to_existing_atom(GwName), AuthnConf); +authn_certs_dir(GwName, AuthnConf) -> + emqx_authentication_config:certs_dir( + emqx_gateway_utils:global_chain(GwName), + AuthnConf + ). + convert_certs(SubDir, Conf) -> convert_certs(<<"dtls_options">>, SubDir, convert_certs(<<"ssl_options">>, SubDir, Conf)). diff --git a/changes/ce/fix-10653.en.md b/changes/ce/fix-10653.en.md new file mode 100644 index 000000000..c18ea9ed0 --- /dev/null +++ b/changes/ce/fix-10653.en.md @@ -0,0 +1 @@ +Store gateway authentication TLS certificates and keys in the data directory. From 609f7bd8fd35b1d76086d7f822deb4e1260b6e47 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 17 Feb 2023 00:16:29 +0200 Subject: [PATCH 040/144] feat(rebalance): port apps from 4.x --- Makefile | 1 + apps/emqx/include/asserts.hrl | 31 + apps/emqx/include/emqx_channel.hrl | 42 + apps/emqx/include/emqx_hooks.hrl | 1 + apps/emqx/priv/bpapi.versions | 5 + apps/emqx/src/emqx_channel.erl | 54 +- apps/emqx/src/emqx_cm.erl | 49 +- apps/emqx/src/emqx_router_helper.erl | 12 +- .../test/emqx_dashboard_api_test_helpers.erl | 6 +- apps/emqx_machine/src/emqx_machine_boot.erl | 10 +- changes/ee/feat-10075.en.md | 1 + changes/ee/feat-10075.zh.md | 1 + lib-ee/emqx_eviction_agent/README.md | 9 + .../etc/emqx_eviction_agent.conf | 3 + .../i18n/emqx_eviction_agent_api_i18n.conf | 14 + lib-ee/emqx_eviction_agent/rebar.config | 2 + .../src/emqx_eviction_agent.app.src | 22 + .../src/emqx_eviction_agent.appup.src | 3 + .../src/emqx_eviction_agent.erl | 346 ++++++++ .../src/emqx_eviction_agent_api.erl | 85 ++ .../src/emqx_eviction_agent_app.erl | 24 + .../src/emqx_eviction_agent_channel.erl | 368 +++++++++ .../src/emqx_eviction_agent_cli.erl | 30 + .../src/emqx_eviction_agent_conn_sup.erl | 21 + .../src/emqx_eviction_agent_sup.erl | 34 + .../proto/emqx_eviction_agent_proto_v1.erl | 27 + .../test/emqx_eviction_agent_SUITE.erl | 403 ++++++++++ .../test/emqx_eviction_agent_api_SUITE.erl | 69 ++ .../emqx_eviction_agent_channel_SUITE.erl | 251 ++++++ .../test/emqx_eviction_agent_cli_SUITE.erl | 39 + .../test/emqx_eviction_agent_test_helpers.erl | 141 ++++ lib-ee/emqx_node_rebalance/README.md | 9 + .../etc/emqx_node_rebalance.conf | 3 + .../i18n/emqx_node_rebalance_api_i18n.conf | 490 ++++++++++++ .../include/emqx_node_rebalance.hrl | 33 + lib-ee/emqx_node_rebalance/rebar.config | 2 + .../src/emqx_node_rebalance.app.src | 22 + .../src/emqx_node_rebalance.appup.src | 3 + .../src/emqx_node_rebalance.erl | 438 +++++++++++ .../src/emqx_node_rebalance_agent.erl | 131 ++++ .../src/emqx_node_rebalance_api.erl | 738 ++++++++++++++++++ .../src/emqx_node_rebalance_app.erl | 22 + .../src/emqx_node_rebalance_cli.erl | 305 ++++++++ .../src/emqx_node_rebalance_evacuation.erl | 308 ++++++++ ...emqx_node_rebalance_evacuation_persist.erl | 120 +++ .../src/emqx_node_rebalance_status.erl | 238 ++++++ .../src/emqx_node_rebalance_sup.erl | 35 + .../emqx_node_rebalance_api_proto_v1.erl | 43 + ...mqx_node_rebalance_evacuation_proto_v1.erl | 22 + .../proto/emqx_node_rebalance_proto_v1.erl | 62 ++ .../emqx_node_rebalance_status_proto_v1.erl | 36 + .../test/emqx_node_rebalance_SUITE.erl | 229 ++++++ .../test/emqx_node_rebalance_agent_SUITE.erl | 214 +++++ .../test/emqx_node_rebalance_api_SUITE.erl | 444 +++++++++++ .../test/emqx_node_rebalance_cli_SUITE.erl | 291 +++++++ .../emqx_node_rebalance_evacuation_SUITE.erl | 271 +++++++ ...ode_rebalance_evacuation_persist_SUITE.erl | 108 +++ mix.exs | 4 +- rebar.config.erl | 4 +- 59 files changed, 6686 insertions(+), 43 deletions(-) create mode 100644 apps/emqx/include/asserts.hrl create mode 100644 apps/emqx/include/emqx_channel.hrl create mode 100644 changes/ee/feat-10075.en.md create mode 100644 changes/ee/feat-10075.zh.md create mode 100644 lib-ee/emqx_eviction_agent/README.md create mode 100644 lib-ee/emqx_eviction_agent/etc/emqx_eviction_agent.conf create mode 100644 lib-ee/emqx_eviction_agent/i18n/emqx_eviction_agent_api_i18n.conf create mode 100644 lib-ee/emqx_eviction_agent/rebar.config create mode 100644 lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.app.src create mode 100644 lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.appup.src create mode 100644 lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.erl create mode 100644 lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_api.erl create mode 100644 lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_app.erl create mode 100644 lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl create mode 100644 lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl create mode 100644 lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl create mode 100644 lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl create mode 100644 lib-ee/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v1.erl create mode 100644 lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl create mode 100644 lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl create mode 100644 lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl create mode 100644 lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl create mode 100644 lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl create mode 100644 lib-ee/emqx_node_rebalance/README.md create mode 100644 lib-ee/emqx_node_rebalance/etc/emqx_node_rebalance.conf create mode 100644 lib-ee/emqx_node_rebalance/i18n/emqx_node_rebalance_api_i18n.conf create mode 100644 lib-ee/emqx_node_rebalance/include/emqx_node_rebalance.hrl create mode 100644 lib-ee/emqx_node_rebalance/rebar.config create mode 100644 lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.app.src create mode 100644 lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.appup.src create mode 100644 lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.erl create mode 100644 lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl create mode 100644 lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_api.erl create mode 100644 lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_app.erl create mode 100644 lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl create mode 100644 lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl create mode 100644 lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl create mode 100644 lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_status.erl create mode 100644 lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl create mode 100644 lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_api_proto_v1.erl create mode 100644 lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_evacuation_proto_v1.erl create mode 100644 lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_proto_v1.erl create mode 100644 lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_status_proto_v1.erl create mode 100644 lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl create mode 100644 lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl create mode 100644 lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl create mode 100644 lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl create mode 100644 lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl create mode 100644 lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl diff --git a/Makefile b/Makefile index 6741317ee..24af58ebc 100644 --- a/Makefile +++ b/Makefile @@ -179,6 +179,7 @@ clean-all: @rm -f rebar.lock @rm -rf deps @rm -rf _build + @rm -f emqx_dialyzer_*_plt .PHONY: deps-all deps-all: $(REBAR) $(PROFILES:%=deps-%) diff --git a/apps/emqx/include/asserts.hrl b/apps/emqx/include/asserts.hrl new file mode 100644 index 000000000..98d8e72fc --- /dev/null +++ b/apps/emqx/include/asserts.hrl @@ -0,0 +1,31 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% This file contains common macros for testing. +%% It must not be used anywhere except in test suites. + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(assertWaitEvent(Code, EventMatch, Timeout), + ?assertMatch( + {_, {ok, EventMatch}}, + ?wait_async_action( + Code, + EventMatch, + Timeout + ) + ) +). diff --git a/apps/emqx/include/emqx_channel.hrl b/apps/emqx/include/emqx_channel.hrl new file mode 100644 index 000000000..d4362633a --- /dev/null +++ b/apps/emqx/include/emqx_channel.hrl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-define(CHANNEL_METRICS, [ + recv_pkt, + recv_msg, + 'recv_msg.qos0', + 'recv_msg.qos1', + 'recv_msg.qos2', + 'recv_msg.dropped', + 'recv_msg.dropped.await_pubrel_timeout', + send_pkt, + send_msg, + 'send_msg.qos0', + 'send_msg.qos1', + 'send_msg.qos2', + 'send_msg.dropped', + 'send_msg.dropped.expired', + 'send_msg.dropped.queue_full', + 'send_msg.dropped.too_large' +]). + +-define(INFO_KEYS, [ + conninfo, + conn_state, + clientinfo, + session, + will_msg +]). diff --git a/apps/emqx/include/emqx_hooks.hrl b/apps/emqx/include/emqx_hooks.hrl index 1665492c5..2373b5928 100644 --- a/apps/emqx/include/emqx_hooks.hrl +++ b/apps/emqx/include/emqx_hooks.hrl @@ -34,6 +34,7 @@ -define(HP_BRIDGE, 870). -define(HP_DELAY_PUB, 860). %% apps that can stop the hooks chain from continuing +-define(HP_NODE_REBALANCE, 110). -define(HP_EXHOOK, 100). %% == Lowest Priority = 0, don't change this value as the plugins may depend on it. diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index db4765e3f..dceb38c47 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -13,6 +13,7 @@ {emqx_conf,2}. {emqx_dashboard,1}. {emqx_delayed,1}. +{emqx_eviction_agent,1}. {emqx_exhook,1}. {emqx_gateway_api_listeners,1}. {emqx_gateway_cm,1}. @@ -26,6 +27,10 @@ {emqx_mgmt_cluster,1}. {emqx_mgmt_trace,1}. {emqx_mgmt_trace,2}. +{emqx_node_rebalance,1}. +{emqx_node_rebalance_api,1}. +{emqx_node_rebalance_evacuation,1}. +{emqx_node_rebalance_status,1}. {emqx_persistent_session,1}. {emqx_plugin_libs,1}. {emqx_plugins,1}. diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 862b72c06..69e0a55f7 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -18,6 +18,7 @@ -module(emqx_channel). -include("emqx.hrl"). +-include("emqx_channel.hrl"). -include("emqx_mqtt.hrl"). -include("logger.hrl"). -include("types.hrl"). @@ -57,6 +58,12 @@ clear_keepalive/1 ]). +%% Export for emqx_channel implementations +-export([ + maybe_nack/1, + maybe_mark_as_delivered/2 +]). + %% Exports for CT -export([set_field/3]). @@ -69,7 +76,7 @@ ] ). --export_type([channel/0, opts/0]). +-export_type([channel/0, opts/0, conn_state/0]). -record(channel, { %% MQTT ConnInfo @@ -131,33 +138,6 @@ quota_timer => expire_quota_limit }). --define(CHANNEL_METRICS, [ - recv_pkt, - recv_msg, - 'recv_msg.qos0', - 'recv_msg.qos1', - 'recv_msg.qos2', - 'recv_msg.dropped', - 'recv_msg.dropped.await_pubrel_timeout', - send_pkt, - send_msg, - 'send_msg.qos0', - 'send_msg.qos1', - 'send_msg.qos2', - 'send_msg.dropped', - 'send_msg.dropped.expired', - 'send_msg.dropped.queue_full', - 'send_msg.dropped.too_large' -]). - --define(INFO_KEYS, [ - conninfo, - conn_state, - clientinfo, - session, - will_msg -]). - -define(LIMITER_ROUTING, message_routing). -dialyzer({no_match, [shutdown/4, ensure_timer/2, interval/2]}). @@ -1078,10 +1058,12 @@ handle_out(unsuback, {PacketId, _ReasonCodes}, Channel) -> handle_out(disconnect, ReasonCode, Channel) when is_integer(ReasonCode) -> ReasonName = disconnect_reason(ReasonCode), handle_out(disconnect, {ReasonCode, ReasonName}, Channel); -handle_out(disconnect, {ReasonCode, ReasonName}, Channel = ?IS_MQTT_V5) -> - Packet = ?DISCONNECT_PACKET(ReasonCode), +handle_out(disconnect, {ReasonCode, ReasonName}, Channel) -> + handle_out(disconnect, {ReasonCode, ReasonName, #{}}, Channel); +handle_out(disconnect, {ReasonCode, ReasonName, Props}, Channel = ?IS_MQTT_V5) -> + Packet = ?DISCONNECT_PACKET(ReasonCode, Props), {ok, [{outgoing, Packet}, {close, ReasonName}], Channel}; -handle_out(disconnect, {_ReasonCode, ReasonName}, Channel) -> +handle_out(disconnect, {_ReasonCode, ReasonName, _Props}, Channel) -> {ok, {close, ReasonName}, Channel}; handle_out(auth, {ReasonCode, Properties}, Channel) -> {ok, ?AUTH_PACKET(ReasonCode, Properties), Channel}; @@ -1198,13 +1180,19 @@ handle_call( {takeover, 'end'}, Channel = #channel{ session = Session, - pendings = Pendings + pendings = Pendings, + conninfo = #{clientid := ClientId} } ) -> ok = emqx_session:takeover(Session), %% TODO: Should not drain deliver here (side effect) Delivers = emqx_utils:drain_deliver(), AllPendings = lists:append(Delivers, Pendings), + ?tp( + debug, + emqx_channel_takeover_end, + #{clientid => ClientId} + ), disconnect_and_shutdown(takenover, AllPendings, Channel); handle_call(list_authz_cache, Channel) -> {reply, emqx_authz_cache:list_authz_cache(), Channel}; @@ -1276,6 +1264,8 @@ handle_info(die_if_test = Info, Channel) -> die_if_test_compiled(), ?SLOG(error, #{msg => "unexpected_info", info => Info}), {ok, Channel}; +handle_info({disconnect, ReasonCode, ReasonName, Props}, Channel) -> + handle_out(disconnect, {ReasonCode, ReasonName, Props}, Channel); handle_info(Info, Channel) -> ?SLOG(error, #{msg => "unexpected_info", info => Info}), {ok, Channel}. diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 0290b57d3..c8296f317 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -23,6 +23,8 @@ -include("logger.hrl"). -include("types.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("stdlib/include/qlc.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). -export([start_link/0]). @@ -72,6 +74,12 @@ get_session_confs/2 ]). +%% Client management +-export([ + channel_with_session_table/1, + live_connection_table/1 +]). + %% gen_server callbacks -export([ init/1, @@ -593,6 +601,40 @@ all_channels() -> Pat = [{{'_', '$1'}, [], ['$1']}], ets:select(?CHAN_TAB, Pat). +%% @doc Get clientinfo for all clients with sessions +channel_with_session_table(ConnModules) -> + Ms = ets:fun2ms( + fun({{ClientId, _ChanPid}, Info, _Stats}) -> + {ClientId, Info} + end + ), + Table = ets:table(?CHAN_INFO_TAB, [{traverse, {select, Ms}}]), + ConnModuleMap = maps:from_list([{Mod, true} || Mod <- ConnModules]), + qlc:q([ + {ClientId, ConnState, ConnInfo, ClientInfo} + || {ClientId, #{ + conn_state := ConnState, + clientinfo := ClientInfo, + conninfo := #{clean_start := false, conn_mod := ConnModule} = ConnInfo + }} <- + Table, + maps:is_key(ConnModule, ConnModuleMap) + ]). + +%% @doc Get all local connection query handle +live_connection_table(ConnModules) -> + Ms = lists:map(fun live_connection_ms/1, ConnModules), + Table = ets:table(?CHAN_CONN_TAB, [{traverse, {select, Ms}}]), + qlc:q([{ClientId, ChanPid} || {ClientId, ChanPid} <- Table, is_channel_connected(ChanPid)]). + +live_connection_ms(ConnModule) -> + {{{'$1', '$2'}, ConnModule}, [], [{{'$1', '$2'}}]}. + +is_channel_connected(ChanPid) when node(ChanPid) =:= node() -> + ets:member(?CHAN_LIVE_TAB, ChanPid); +is_channel_connected(_ChanPid) -> + false. + %% @doc Get all registered clientIDs. Debug/test interface all_client_ids() -> Pat = [{{'$1', '_'}, [], ['$1']}], @@ -693,7 +735,8 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- clean_down({ChanPid, ClientId}) -> - do_unregister_channel({ClientId, ChanPid}). + do_unregister_channel({ClientId, ChanPid}), + ok = ?tp(debug, emqx_cm_clean_down, #{client_id => ClientId}). stats_fun() -> lists:foreach(fun update_stats/1, ?CHAN_STATS). @@ -719,12 +762,12 @@ get_chann_conn_mod(ClientId, ChanPid) -> wrap_rpc(emqx_cm_proto_v1:get_chann_conn_mod(ClientId, ChanPid)). mark_channel_connected(ChanPid) -> - ?tp(emqx_cm_connected_client_count_inc, #{}), + ?tp(emqx_cm_connected_client_count_inc, #{chan_pid => ChanPid}), ets:insert_new(?CHAN_LIVE_TAB, {ChanPid, true}), ok. mark_channel_disconnected(ChanPid) -> - ?tp(emqx_cm_connected_client_count_dec, #{}), + ?tp(emqx_cm_connected_client_count_dec, #{chan_pid => ChanPid}), ets:delete(?CHAN_LIVE_TAB, ChanPid), ok. diff --git a/apps/emqx/src/emqx_router_helper.erl b/apps/emqx/src/emqx_router_helper.erl index e2d54b99e..4bff98072 100644 --- a/apps/emqx/src/emqx_router_helper.erl +++ b/apps/emqx/src/emqx_router_helper.erl @@ -167,9 +167,15 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, _State) -> - ok = ekka:unmonitor(membership), - emqx_stats:cancel_update(route_stats), - mnesia:unsubscribe({table, ?ROUTING_NODE, simple}). + try + ok = ekka:unmonitor(membership), + emqx_stats:cancel_update(route_stats), + mnesia:unsubscribe({table, ?ROUTING_NODE, simple}) + catch + exit:{noproc, {gen_server, call, [mria_membership, _]}} -> + ?SLOG(warning, #{msg => "mria_membership_down"}), + ok + end. code_change(_OldVsn, State, _Extra) -> {ok, State}. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl index 91c7729d3..25b4065de 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl @@ -20,6 +20,7 @@ set_default_config/0, set_default_config/1, set_default_config/2, + set_default_config/3, request/2, request/3, request/4, @@ -40,11 +41,14 @@ set_default_config(DefaultUsername) -> set_default_config(DefaultUsername, false). set_default_config(DefaultUsername, HAProxyEnabled) -> + set_default_config(DefaultUsername, HAProxyEnabled, #{}). + +set_default_config(DefaultUsername, HAProxyEnabled, Opts) -> Config = #{ listeners => #{ http => #{ enable => true, - bind => 18083, + bind => maps:get(bind, Opts, 18083), inet6 => false, ipv6_v6only => false, max_connections => 512, diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 82b3d602f..e3f84079b 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -149,8 +149,14 @@ basic_reboot_apps() -> emqx_plugins ], case emqx_release:edition() of - ce -> CE; - ee -> CE ++ [] + ce -> + CE; + ee -> + CE ++ + [ + emqx_eviction_agent, + emqx_node_rebalance + ] end. sorted_reboot_apps() -> diff --git a/changes/ee/feat-10075.en.md b/changes/ee/feat-10075.en.md new file mode 100644 index 000000000..e6e070ddc --- /dev/null +++ b/changes/ee/feat-10075.en.md @@ -0,0 +1 @@ +Add node rebalance/node evacuation [functionality](https://github.com/emqx/eip/blob/main/active/0020-node-rebalance.md). diff --git a/changes/ee/feat-10075.zh.md b/changes/ee/feat-10075.zh.md new file mode 100644 index 000000000..36c78acb8 --- /dev/null +++ b/changes/ee/feat-10075.zh.md @@ -0,0 +1 @@ +添加节点再平衡/节点疏散[功能](https://github.com/emqx/eip/blob/main/active/0020-node-rebalance.md)。 diff --git a/lib-ee/emqx_eviction_agent/README.md b/lib-ee/emqx_eviction_agent/README.md new file mode 100644 index 000000000..f9b8037bf --- /dev/null +++ b/lib-ee/emqx_eviction_agent/README.md @@ -0,0 +1,9 @@ +emqx_eviction_agent +===== + +An OTP library + +Build +----- + + $ rebar3 compile diff --git a/lib-ee/emqx_eviction_agent/etc/emqx_eviction_agent.conf b/lib-ee/emqx_eviction_agent/etc/emqx_eviction_agent.conf new file mode 100644 index 000000000..011b7fb0f --- /dev/null +++ b/lib-ee/emqx_eviction_agent/etc/emqx_eviction_agent.conf @@ -0,0 +1,3 @@ +##-------------------------------------------------------------------- +## EMQX Eviction Agent Plugin +##-------------------------------------------------------------------- diff --git a/lib-ee/emqx_eviction_agent/i18n/emqx_eviction_agent_api_i18n.conf b/lib-ee/emqx_eviction_agent/i18n/emqx_eviction_agent_api_i18n.conf new file mode 100644 index 000000000..8bb7282c3 --- /dev/null +++ b/lib-ee/emqx_eviction_agent/i18n/emqx_eviction_agent_api_i18n.conf @@ -0,0 +1,14 @@ +emqx_eviction_agent_api { + + node_eviction_status_get { + desc { + en: "Get the node eviction status" + zh: "获取节点驱逐状态" + } + label { + en: "Node Eviction Status" + zh: "节点驱逐状态" + } + } + +} diff --git a/lib-ee/emqx_eviction_agent/rebar.config b/lib-ee/emqx_eviction_agent/rebar.config new file mode 100644 index 000000000..b055d8f4f --- /dev/null +++ b/lib-ee/emqx_eviction_agent/rebar.config @@ -0,0 +1,2 @@ +{deps, [{emqx, {path, "../../apps/emqx"}}]}. +{project_plugins, [erlfmt]}. diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.app.src b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.app.src new file mode 100644 index 000000000..a360133f4 --- /dev/null +++ b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.app.src @@ -0,0 +1,22 @@ +{application, emqx_eviction_agent, [ + {description, "EMQX Eviction Agent"}, + {vsn, "5.0.0"}, + {registered, [ + emqx_eviction_agent_sup, + emqx_eviction_agent, + emqx_eviction_agent_conn_sup + ]}, + {applications, [ + kernel, + stdlib, + emqx_ctl + ]}, + {mod, {emqx_eviction_agent_app, []}}, + {env, []}, + {modules, []}, + {maintainers, ["EMQX Team "]}, + {links, [ + {"Homepage", "https://emqx.io/"}, + {"Github", "https://github.com/emqx"} + ]} +]}. diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.appup.src b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.appup.src new file mode 100644 index 000000000..c1b84778d --- /dev/null +++ b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.appup.src @@ -0,0 +1,3 @@ +%% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! +{VSN, [{<<".*">>, []}], [{<<".*">>, []}]}. diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.erl b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.erl new file mode 100644 index 000000000..b8e1b5236 --- /dev/null +++ b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.erl @@ -0,0 +1,346 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). +-include_lib("emqx/include/emqx_hooks.hrl"). + +-include_lib("stdlib/include/qlc.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-export([ + start_link/0, + enable/2, + disable/1, + status/0, + connection_count/0, + session_count/0, + session_count/1, + evict_connections/1, + evict_sessions/2, + evict_sessions/3, + evict_session_channel/3 +]). + +-behaviour(gen_server). + +-export([ + init/1, + handle_call/3, + handle_info/2, + handle_cast/2, + code_change/3 +]). + +-export([ + on_connect/2, + on_connack/3 +]). + +-export([ + hook/0, + unhook/0 +]). + +-export_type([server_reference/0]). + +-define(CONN_MODULES, [emqx_connection, emqx_ws_connection, emqx_eviction_agent_channel]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +-type server_reference() :: binary() | undefined. +-type status() :: {enabled, conn_stats()} | disabled. +-type conn_stats() :: #{ + connections := non_neg_integer(), + sessions := non_neg_integer() +}. +-type kind() :: atom(). + +-spec start_link() -> startlink_ret(). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec enable(kind(), server_reference()) -> ok_or_error(eviction_agent_busy). +enable(Kind, ServerReference) -> + gen_server:call(?MODULE, {enable, Kind, ServerReference}). + +-spec disable(kind()) -> ok. +disable(Kind) -> + gen_server:call(?MODULE, {disable, Kind}). + +-spec status() -> status(). +status() -> + case enable_status() of + {enabled, _Kind, _ServerReference} -> + {enabled, stats()}; + disabled -> + disabled + end. + +-spec evict_connections(pos_integer()) -> ok_or_error(disabled). +evict_connections(N) -> + case enable_status() of + {enabled, _Kind, ServerReference} -> + ok = do_evict_connections(N, ServerReference); + disabled -> + {error, disabled} + end. + +-spec evict_sessions(pos_integer(), node() | [node()]) -> ok_or_error(disabled). +evict_sessions(N, Node) when is_atom(Node) -> + evict_sessions(N, [Node]); +evict_sessions(N, Nodes) when is_list(Nodes) andalso length(Nodes) > 0 -> + evict_sessions(N, Nodes, any). + +-spec evict_sessions(pos_integer(), node() | [node()], atom()) -> ok_or_error(disabled). +evict_sessions(N, Node, ConnState) when is_atom(Node) -> + evict_sessions(N, [Node], ConnState); +evict_sessions(N, Nodes, ConnState) when + is_list(Nodes) andalso length(Nodes) > 0 +-> + case enable_status() of + {enabled, _Kind, _ServerReference} -> + ok = do_evict_sessions(N, Nodes, ConnState); + disabled -> + {error, disabled} + end. + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([]) -> + _ = persistent_term:erase(?MODULE), + {ok, #{}}. + +%% enable +handle_call({enable, Kind, ServerReference}, _From, St) -> + Reply = + case enable_status() of + disabled -> + ok = persistent_term:put(?MODULE, {enabled, Kind, ServerReference}); + {enabled, Kind, _ServerReference} -> + ok = persistent_term:put(?MODULE, {enabled, Kind, ServerReference}); + {enabled, _OtherKind, _ServerReference} -> + {error, eviction_agent_busy} + end, + {reply, Reply, St}; +%% disable +handle_call({disable, Kind}, _From, St) -> + Reply = + case enable_status() of + disabled -> + {error, disabled}; + {enabled, Kind, _ServerReference} -> + _ = persistent_term:erase(?MODULE), + ok; + {enabled, _OtherKind, _ServerReference} -> + {error, eviction_agent_busy} + end, + {reply, Reply, St}; +handle_call(Msg, _From, St) -> + ?SLOG(warning, #{msg => "unknown_call", call => Msg, state => St}), + {reply, {error, unknown_call}, St}. + +handle_info(Msg, St) -> + ?SLOG(warning, #{msg => "unknown_msg", info => Msg, state => St}), + {noreply, St}. + +handle_cast(Msg, St) -> + ?SLOG(warning, #{msg => "unknown_cast", cast => Msg, state => St}), + {noreply, St}. + +code_change(_Vsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Hook callbacks +%%-------------------------------------------------------------------- + +on_connect(_ConnInfo, _Props) -> + case enable_status() of + {enabled, _Kind, _ServerReference} -> + {stop, {error, ?RC_USE_ANOTHER_SERVER}}; + disabled -> + ignore + end. + +on_connack( + #{proto_name := <<"MQTT">>, proto_ver := ?MQTT_PROTO_V5}, + use_another_server, + Props +) -> + case enable_status() of + {enabled, _Kind, ServerReference} -> + {ok, Props#{'Server-Reference' => ServerReference}}; + disabled -> + {ok, Props} + end; +on_connack(_ClientInfo, _Reason, Props) -> + {ok, Props}. + +%%-------------------------------------------------------------------- +%% Hook funcs +%%-------------------------------------------------------------------- + +hook() -> + ?tp(debug, eviction_agent_hook, #{}), + ok = emqx_hooks:put('client.connack', {?MODULE, on_connack, []}, ?HP_NODE_REBALANCE), + ok = emqx_hooks:put('client.connect', {?MODULE, on_connect, []}, ?HP_NODE_REBALANCE). + +unhook() -> + ?tp(debug, eviction_agent_unhook, #{}), + ok = emqx_hooks:del('client.connect', {?MODULE, on_connect}), + ok = emqx_hooks:del('client.connack', {?MODULE, on_connack}). + +enable_status() -> + persistent_term:get(?MODULE, disabled). + +% connection management +stats() -> + #{ + connections => connection_count(), + sessions => session_count() + }. + +connection_table() -> + emqx_cm:live_connection_table(?CONN_MODULES). + +connection_count() -> + table_count(connection_table()). + +channel_with_session_table(any) -> + qlc:q([ + {ClientId, ConnInfo, ClientInfo} + || {ClientId, _, ConnInfo, ClientInfo} <- + emqx_cm:channel_with_session_table(?CONN_MODULES) + ]); +channel_with_session_table(RequiredConnState) -> + qlc:q([ + {ClientId, ConnInfo, ClientInfo} + || {ClientId, ConnState, ConnInfo, ClientInfo} <- + emqx_cm:channel_with_session_table(?CONN_MODULES), + RequiredConnState =:= ConnState + ]). + +session_count() -> + session_count(any). + +session_count(ConnState) -> + table_count(channel_with_session_table(ConnState)). + +table_count(QH) -> + qlc:fold(fun(_, Acc) -> Acc + 1 end, 0, QH). + +take_connections(N) -> + ChanQH = qlc:q([ChanPid || {_ClientId, ChanPid} <- connection_table()]), + ChanPidCursor = qlc:cursor(ChanQH), + ChanPids = qlc:next_answers(ChanPidCursor, N), + ok = qlc:delete_cursor(ChanPidCursor), + ChanPids. + +take_channel_with_sessions(N, ConnState) -> + ChanPidCursor = qlc:cursor(channel_with_session_table(ConnState)), + Channels = qlc:next_answers(ChanPidCursor, N), + ok = qlc:delete_cursor(ChanPidCursor), + Channels. + +do_evict_connections(N, ServerReference) when N > 0 -> + ChanPids = take_connections(N), + ok = lists:foreach( + fun(ChanPid) -> + disconnect_channel(ChanPid, ServerReference) + end, + ChanPids + ). + +do_evict_sessions(N, Nodes, ConnState) when N > 0 -> + Channels = take_channel_with_sessions(N, ConnState), + ok = lists:foreach( + fun({ClientId, ConnInfo, ClientInfo}) -> + evict_session_channel(Nodes, ClientId, ConnInfo, ClientInfo) + end, + Channels + ). + +evict_session_channel(Nodes, ClientId, ConnInfo, ClientInfo) -> + Node = select_random(Nodes), + ?SLOG( + info, + #{ + msg => "evict_session_channel", + client_id => ClientId, + node => Node, + conn_info => ConnInfo, + client_info => ClientInfo + } + ), + case emqx_eviction_agent_proto_v1:evict_session_channel(Node, ClientId, ConnInfo, ClientInfo) of + {badrpc, Reason} -> + ?SLOG( + error, + #{ + msg => "evict_session_channel_rpc_error", + client_id => ClientId, + node => Node, + reason => Reason + } + ), + {error, Reason}; + {error, Reason} = Error -> + ?SLOG( + error, + #{ + msg => "evict_session_channel_error", + client_id => ClientId, + node => Node, + reason => Reason + } + ), + Error; + Res -> + Res + end. + +-spec evict_session_channel( + emqx_types:clientid(), + emqx_types:conninfo(), + emqx_types:clientinfo() +) -> supervisor:startchild_ret(). +evict_session_channel(ClientId, ConnInfo, ClientInfo) -> + ?SLOG(info, #{ + msg => "evict_session_channel", + client_id => ClientId, + conn_info => ConnInfo, + client_info => ClientInfo + }), + Result = emqx_eviction_agent_channel:start_supervised( + #{ + conninfo => ConnInfo, + clientinfo => ClientInfo + } + ), + ?SLOG( + info, + #{ + msg => "evict_session_channel_result", + client_id => ClientId, + result => Result + } + ), + Result. + +disconnect_channel(ChanPid, ServerReference) -> + ChanPid ! + {disconnect, ?RC_USE_ANOTHER_SERVER, use_another_server, #{ + 'Server-Reference' => ServerReference + }}. + +select_random(List) when length(List) > 0 -> + lists:nth(rand:uniform(length(List)), List). diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_api.erl b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_api.erl new file mode 100644 index 000000000..d8c1d7645 --- /dev/null +++ b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_api.erl @@ -0,0 +1,85 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_api). + +-behaviour(minirest_api). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% Swagger specs from hocon schema +-export([ + api_spec/0, + paths/0, + schema/1, + namespace/0 +]). + +-export([ + fields/1, + roots/0 +]). + +%% API callbacks +-export([ + '/node_eviction/status'/2 +]). + +-import(hoconsc, [mk/2, ref/1, ref/2]). + +namespace() -> "node_eviction". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + [ + "/node_eviction/status" + ]. + +schema("/node_eviction/status") -> + #{ + 'operationId' => '/node_eviction/status', + get => #{ + tags => [<<"node_eviction">>], + summary => <<"Get node eviction status">>, + description => ?DESC("node_eviction_status_get"), + responses => #{ + 200 => schema_status() + } + } + }. + +'/node_eviction/status'(_Bindings, _Params) -> + case emqx_eviction_agent:status() of + disabled -> + {200, #{status => disabled}}; + {enabled, Stats} -> + {200, #{ + status => enabled, + stats => Stats + }} + end. + +schema_status() -> + mk(hoconsc:union([ref(status_enabled), ref(status_disabled)]), #{}). + +roots() -> []. + +fields(status_enabled) -> + [ + {status, mk(enabled, #{default => enabled})}, + {stats, ref(stats)} + ]; +fields(stats) -> + [ + {connections, mk(integer(), #{})}, + {sessions, mk(integer(), #{})} + ]; +fields(status_disabled) -> + [ + {status, mk(disabled, #{default => disabled})} + ]. diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_app.erl b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_app.erl new file mode 100644 index 000000000..63af59b09 --- /dev/null +++ b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_app.erl @@ -0,0 +1,24 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_app). + +-behaviour(application). + +-emqx_plugin(?MODULE). + +-export([ + start/2, + stop/1 +]). + +start(_Type, _Args) -> + ok = emqx_eviction_agent:hook(), + {ok, Sup} = emqx_eviction_agent_sup:start_link(), + ok = emqx_eviction_agent_cli:load(), + {ok, Sup}. + +stop(_State) -> + ok = emqx_eviction_agent:unhook(), + ok = emqx_eviction_agent_cli:unload(). diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl new file mode 100644 index 000000000..a42033c0f --- /dev/null +++ b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl @@ -0,0 +1,368 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%% MQTT Channel +-module(emqx_eviction_agent_channel). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_channel.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-logger_header("[Evicted Channel]"). + +-export([ + start_link/1, + start_supervised/1, + call/2, + call/3, + cast/2, + stop/1 +]). + +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). + +-import( + emqx_misc, + [ + maybe_apply/2 + ] +). + +-type opts() :: #{ + conninfo := emqx_types:conninfo(), + clientinfo := emqx_types:clientinfo() +}. + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec start_supervised(opts()) -> supervisor:startchild_ret(). +start_supervised(#{clientinfo := #{clientid := ClientId}} = Opts) -> + RandomId = integer_to_binary(erlang:unique_integer([positive])), + ClientIdBin = bin_clientid(ClientId), + Id = <>, + ChildSpec = #{ + id => Id, + start => {?MODULE, start_link, [Opts]}, + restart => temporary, + shutdown => 5000, + type => worker, + modules => [?MODULE] + }, + supervisor:start_child( + emqx_eviction_agent_conn_sup, + ChildSpec + ). + +-spec start_link(opts()) -> startlink_ret(). +start_link(Opts) -> + gen_server:start_link(?MODULE, [Opts], []). + +-spec cast(pid(), term()) -> ok. +cast(Pid, Req) -> + gen_server:cast(Pid, Req). + +-spec call(pid(), term()) -> term(). +call(Pid, Req) -> + call(Pid, Req, infinity). + +-spec call(pid(), term(), timeout()) -> term(). +call(Pid, Req, Timeout) -> + gen_server:call(Pid, Req, Timeout). + +-spec stop(pid()) -> ok. +stop(Pid) -> + gen_server:stop(Pid). + +%%-------------------------------------------------------------------- +%% gen_server API +%%-------------------------------------------------------------------- + +init([#{conninfo := OldConnInfo, clientinfo := #{clientid := ClientId} = OldClientInfo}]) -> + process_flag(trap_exit, true), + ClientInfo = clientinfo(OldClientInfo), + ConnInfo = conninfo(OldConnInfo), + case open_session(ConnInfo, ClientInfo) of + {ok, Channel0} -> + case set_expiry_timer(Channel0) of + {ok, Channel1} -> + ?SLOG( + info, + #{ + msg => "channel_initialized", + clientid => ClientId, + node => node() + } + ), + ok = emqx_cm:mark_channel_disconnected(self()), + {ok, Channel1, hibernate}; + {error, Reason} -> + {stop, Reason} + end; + {error, Reason} -> + {stop, Reason} + end. + +handle_call(kick, _From, Channel) -> + {stop, kicked, ok, Channel}; +handle_call(discard, _From, Channel) -> + {stop, discarded, ok, Channel}; +handle_call({takeover, 'begin'}, _From, #{session := Session} = Channel) -> + {reply, Session, Channel#{takeover => true}}; +handle_call( + {takeover, 'end'}, + _From, + #{ + session := Session, + clientinfo := #{clientid := ClientId}, + pendings := Pendings + } = Channel +) -> + ok = emqx_session:takeover(Session), + %% TODO: Should not drain deliver here (side effect) + Delivers = emqx_misc:drain_deliver(), + AllPendings = lists:append(Delivers, Pendings), + ?tp( + debug, + emqx_channel_takeover_end, + #{clientid => ClientId} + ), + {stop, normal, AllPendings, Channel}; +handle_call(list_acl_cache, _From, Channel) -> + {reply, [], Channel}; +handle_call({quota, _Policy}, _From, Channel) -> + {reply, ok, Channel}; +handle_call(Req, _From, Channel) -> + ?SLOG( + error, + #{ + msg => "unexpected_call", + req => Req + } + ), + {reply, ignored, Channel}. + +handle_info(Deliver = {deliver, _Topic, _Msg}, Channel) -> + Delivers = [Deliver | emqx_misc:drain_deliver()], + {noreply, handle_deliver(Delivers, Channel)}; +handle_info(expire_session, Channel) -> + {stop, expired, Channel}; +handle_info(Info, Channel) -> + ?SLOG( + error, + #{ + msg => "unexpected_info", + info => Info + } + ), + {noreply, Channel}. + +handle_cast(Msg, Channel) -> + ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), + {noreply, Channel}. + +terminate(Reason, #{conninfo := ConnInfo, clientinfo := ClientInfo, session := Session} = Channel) -> + ok = cancel_expiry_timer(Channel), + (Reason =:= expired) andalso emqx_persistent_session:persist(ClientInfo, ConnInfo, Session), + emqx_session:terminate(ClientInfo, Reason, Session). + +code_change(_OldVsn, Channel, _Extra) -> + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +%% TODO: sync with emqx_channel +handle_deliver( + Delivers, + #{ + takeover := true, + pendings := Pendings, + session := Session, + clientinfo := #{clientid := ClientId} = ClientInfo + } = Channel +) -> + %% NOTE: Order is important here. While the takeover is in + %% progress, the session cannot enqueue messages, since it already + %% passed on the queue to the new connection in the session state. + NPendings = lists:append( + Pendings, + emqx_session:ignore_local(ClientInfo, emqx_channel:maybe_nack(Delivers), ClientId, Session) + ), + Channel#{pendings => NPendings}; +handle_deliver( + Delivers, + #{ + takeover := false, + session := Session, + clientinfo := #{clientid := ClientId} = ClientInfo + } = Channel +) -> + Delivers1 = emqx_channel:maybe_nack(Delivers), + Delivers2 = emqx_session:ignore_local(ClientInfo, Delivers1, ClientId, Session), + NSession = emqx_session:enqueue(ClientInfo, Delivers2, Session), + NChannel = persist(NSession, Channel), + %% We consider queued/dropped messages as delivered since they are now in the session state. + emqx_channel:maybe_mark_as_delivered(Session, Delivers), + NChannel. + +cancel_expiry_timer(#{expiry_timer := TRef}) when is_reference(TRef) -> + _ = erlang:cancel_timer(TRef), + ok; +cancel_expiry_timer(_) -> + ok. + +set_expiry_timer(#{conninfo := ConnInfo} = Channel) -> + case maps:get(expiry_interval, ConnInfo) of + ?UINT_MAX -> + {ok, Channel}; + I when I > 0 -> + Timer = erlang:send_after(timer:seconds(I), self(), expire_session), + {ok, Channel#{expiry_timer => Timer}}; + _ -> + {error, should_be_expired} + end. + +open_session(ConnInfo, #{clientid := ClientId} = ClientInfo) -> + Channel = channel(ConnInfo, ClientInfo), + case emqx_cm:open_session(false, ClientInfo, ConnInfo) of + {ok, #{present := false}} -> + ?SLOG( + info, + #{ + msg => "no_session", + clientid => ClientId, + node => node() + } + ), + {error, no_session}; + {ok, #{session := Session, present := true, pendings := Pendings0}} -> + ?SLOG( + info, + #{ + msg => "session_opened", + clientid => ClientId, + node => node() + } + ), + Pendings1 = lists:usort(lists:append(Pendings0, emqx_misc:drain_deliver())), + NSession = emqx_session:enqueue( + ClientInfo, + emqx_session:ignore_local( + ClientInfo, + emqx_channel:maybe_nack(Pendings1), + ClientId, + Session + ), + Session + ), + NChannel = Channel#{session => NSession}, + ok = emqx_cm:insert_channel_info(ClientId, info(NChannel), stats(NChannel)), + ?SLOG( + info, + #{ + msg => "channel_info_updated", + clientid => ClientId, + node => node() + } + ), + {ok, NChannel}; + {error, Reason} = Error -> + ?SLOG( + error, + #{ + msg => "session_open_failed", + clientid => ClientId, + node => node(), + reason => Reason + } + ), + Error + end. + +conninfo(OldConnInfo) -> + DisconnectedAt = maps:get(disconnected_at, OldConnInfo, erlang:system_time(millisecond)), + ConnInfo0 = maps:with( + [ + socktype, + sockname, + peername, + peercert, + clientid, + clean_start, + receive_maximum, + expiry_interval, + connected_at, + disconnected_at, + keepalive + ], + OldConnInfo + ), + ConnInfo0#{ + conn_mod => ?MODULE, + connected => false, + disconnected_at => DisconnectedAt + }. + +clientinfo(OldClientInfo) -> + maps:with( + [ + zone, + protocol, + peerhost, + sockport, + clientid, + username, + is_bridge, + is_superuser, + mountpoint + ], + OldClientInfo + ). + +channel(ConnInfo, ClientInfo) -> + #{ + conninfo => ConnInfo, + clientinfo => ClientInfo, + expiry_timer => undefined, + takeover => false, + resuming => false, + pendings => [] + }. + +persist(Session, #{clientinfo := ClientInfo, conninfo := ConnInfo} = Channel) -> + Session1 = emqx_persistent_session:persist(ClientInfo, ConnInfo, Session), + Channel#{session => Session1}. + +info(Channel) -> + #{ + conninfo => maps:get(conninfo, Channel, undefined), + clientinfo => maps:get(clientinfo, Channel, undefined), + session => maybe_apply( + fun emqx_session:info/1, + maps:get(session, Channel, undefined) + ), + conn_state => disconnected + }. + +stats(#{session := Session}) -> + lists:append(emqx_session:stats(Session), emqx_pd:get_counters(?CHANNEL_METRICS)). + +bin_clientid(ClientId) when is_binary(ClientId) -> + ClientId; +bin_clientid(ClientId) when is_atom(ClientId) -> + atom_to_binary(ClientId). diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl new file mode 100644 index 000000000..3ae9365e3 --- /dev/null +++ b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl @@ -0,0 +1,30 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_cli). + +%% APIs +-export([ + load/0, + unload/0, + cli/1 +]). + +load() -> + emqx_ctl:register_command(eviction, {?MODULE, cli}, []). + +unload() -> + emqx_ctl:unregister_command(eviction). + +cli(["status"]) -> + case emqx_eviction_agent:status() of + disabled -> + emqx_ctl:print("Eviction status: disabled~n"); + {enabled, _Stats} -> + emqx_ctl:print("Eviction status: enabled~n") + end; +cli(_) -> + emqx_ctl:usage( + [{"eviction status", "Get current node eviction status"}] + ). diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl new file mode 100644 index 000000000..195555bd3 --- /dev/null +++ b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl @@ -0,0 +1,21 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_conn_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + {ok, + { + #{strategy => one_for_one, intensity => 10, period => 3600}, + [] + }}. diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl new file mode 100644 index 000000000..8b774ef85 --- /dev/null +++ b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl @@ -0,0 +1,34 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + Childs = [ + child_spec(worker, emqx_eviction_agent, []), + child_spec(supervisor, emqx_eviction_agent_conn_sup, []) + ], + {ok, { + #{strategy => one_for_one, intensity => 10, period => 3600}, + Childs + }}. + +child_spec(Type, Mod, Args) -> + #{ + id => Mod, + start => {Mod, start_link, Args}, + restart => permanent, + shutdown => 5000, + type => Type, + modules => [Mod] + }. diff --git a/lib-ee/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v1.erl b/lib-ee/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v1.erl new file mode 100644 index 000000000..f4c958150 --- /dev/null +++ b/lib-ee/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v1.erl @@ -0,0 +1,27 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_proto_v1). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + evict_session_channel/4 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.22". + +-spec evict_session_channel( + node(), + emqx_types:clientid(), + emqx_types:conninfo(), + emqx_types:clientinfo() +) -> supervisor:startchild_err() | emqx_rpc:badrpc(). +evict_session_channel(Node, ClientId, ConnInfo, ClientInfo) -> + rpc:call(Node, emqx_eviction_agent, evict_session_channel, [ClientId, ConnInfo, ClientInfo]). diff --git a/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl b/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl new file mode 100644 index 000000000..0574ccec3 --- /dev/null +++ b/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl @@ -0,0 +1,403 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/asserts.hrl"). + +-import( + emqx_eviction_agent_test_helpers, + [emqtt_connect/0, emqtt_connect/1, emqtt_connect/2] +). + +-define(assertPrinted(Printed, Code), + ?assertMatch( + {match, _}, + re:run(Code, Printed) + ) +). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_common_test_helpers:start_apps([emqx_eviction_agent]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([emqx_eviction_agent]). + +init_per_testcase(Case, Config) -> + _ = emqx_eviction_agent:disable(test_eviction), + ok = snabbkaffe:start_trace(), + start_slave(Case, Config). + +start_slave(t_explicit_session_takeover, Config) -> + ClusterNodes = emqx_eviction_agent_test_helpers:start_cluster( + [{evacuate_test1, 2883}, {evacuate_test2, 3883}], + [emqx_eviction_agent] + ), + [{evacuate_nodes, ClusterNodes} | Config]; +start_slave(_Case, Config) -> + Config. + +end_per_testcase(TestCase, Config) -> + emqx_eviction_agent:disable(test_eviction), + ok = snabbkaffe:stop(), + stop_slave(TestCase, Config). + +stop_slave(t_explicit_session_takeover, Config) -> + emqx_eviction_agent_test_helpers:stop_cluster( + ?config(evacuate_nodes, Config), + [emqx_eviction_agent] + ); +stop_slave(_Case, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_enable_disable(_Config) -> + erlang:process_flag(trap_exit, true), + + ?assertMatch( + disabled, + emqx_eviction_agent:status() + ), + + {ok, C0} = emqtt_connect(), + ok = emqtt:disconnect(C0), + + ok = emqx_eviction_agent:enable(test_eviction, undefined), + + ?assertMatch( + {error, eviction_agent_busy}, + emqx_eviction_agent:enable(bar, undefined) + ), + + ?assertMatch( + ok, + emqx_eviction_agent:enable(test_eviction, <<"srv">>) + ), + + ?assertMatch( + {enabled, #{}}, + emqx_eviction_agent:status() + ), + + ?assertMatch( + {error, {use_another_server, #{}}}, + emqtt_connect() + ), + + ?assertMatch( + {error, eviction_agent_busy}, + emqx_eviction_agent:disable(bar) + ), + + ?assertMatch( + ok, + emqx_eviction_agent:disable(test_eviction) + ), + + ?assertMatch( + {error, disabled}, + emqx_eviction_agent:disable(test_eviction) + ), + + ?assertMatch( + disabled, + emqx_eviction_agent:status() + ), + + {ok, C1} = emqtt_connect(), + ok = emqtt:disconnect(C1). + +t_evict_connections_status(_Config) -> + erlang:process_flag(trap_exit, true), + + {ok, _C} = emqtt_connect(), + + {error, disabled} = emqx_eviction_agent:evict_connections(1), + + ok = emqx_eviction_agent:enable(test_eviction, undefined), + + ?assertMatch( + {enabled, #{connections := 1, sessions := _}}, + emqx_eviction_agent:status() + ), + + ok = emqx_eviction_agent:evict_connections(1), + + ct:sleep(100), + + ?assertMatch( + {enabled, #{connections := 0, sessions := _}}, + emqx_eviction_agent:status() + ), + + ok = emqx_eviction_agent:disable(test_eviction). + +t_explicit_session_takeover(Config) -> + _ = erlang:process_flag(trap_exit, true), + ok = restart_emqx(), + + [{Node1, Port1}, {Node2, _Port2}] = ?config(evacuate_nodes, Config), + + {ok, C0} = emqtt_connect([ + {clientid, <<"client_with_session">>}, + {clean_start, false}, + {port, Port1} + ]), + {ok, _, _} = emqtt:subscribe(C0, <<"t1">>), + + ok = rpc:call(Node1, emqx_eviction_agent, enable, [test_eviction, undefined]), + + ?assertEqual( + 1, + rpc:call(Node1, emqx_eviction_agent, connection_count, []) + ), + + [ChanPid] = rpc:call(Node1, emqx_cm, lookup_channels, [<<"client_with_session">>]), + + ?assertWaitEvent( + begin + ok = rpc:call(Node1, emqx_eviction_agent, evict_connections, [1]), + receive + {'EXIT', C0, {disconnected, ?RC_USE_ANOTHER_SERVER, _}} -> ok + after 1000 -> + ?assert(false, "Connection not evicted") + end + end, + #{?snk_kind := emqx_cm_connected_client_count_dec, chan_pid := ChanPid}, + 2000 + ), + + ?assertEqual( + 0, + rpc:call(Node1, emqx_eviction_agent, connection_count, []) + ), + + ?assertEqual( + 1, + rpc:call(Node1, emqx_eviction_agent, session_count, []) + ), + + %% First, evacuate to the same node + + ?assertWaitEvent( + rpc:call(Node1, emqx_eviction_agent, evict_sessions, [1, Node1]), + #{?snk_kind := emqx_channel_takeover_end, clientid := <<"client_with_session">>}, + 1000 + ), + + ok = rpc:call(Node1, emqx_eviction_agent, disable, [test_eviction]), + + {ok, C1} = emqtt_connect([{port, Port1}]), + emqtt:publish(C1, <<"t1">>, <<"MessageToEvictedSession1">>), + ok = emqtt:disconnect(C1), + + ok = rpc:call(Node1, emqx_eviction_agent, enable, [test_eviction, undefined]), + + %% Evacuate to another node + + ?assertWaitEvent( + rpc:call(Node1, emqx_eviction_agent, evict_sessions, [1, Node2]), + #{?snk_kind := emqx_channel_takeover_end, clientid := <<"client_with_session">>}, + 1000 + ), + + ?assertEqual( + 0, + rpc:call(Node1, emqx_eviction_agent, session_count, []) + ), + + ?assertEqual( + 1, + rpc:call(Node2, emqx_eviction_agent, session_count, []) + ), + + ok = rpc:call(Node1, emqx_eviction_agent, disable, [test_eviction]), + + %% Session is on Node2, but we connect to Node1 + {ok, C2} = emqtt_connect([{port, Port1}]), + emqtt:publish(C2, <<"t1">>, <<"MessageToEvictedSession2">>), + ok = emqtt:disconnect(C2), + + ct:sleep(100), + + %% Session is on Node2, but we connect the subscribed client to Node1 + %% It should take over the session for the third time and recieve + %% previously published messages + {ok, C3} = emqtt_connect([ + {clientid, <<"client_with_session">>}, + {clean_start, false}, + {port, Port1} + ]), + + ok = assert_receive_publish( + [ + #{payload => <<"MessageToEvictedSession1">>, topic => <<"t1">>}, + #{payload => <<"MessageToEvictedSession2">>, topic => <<"t1">>} + ] + ), + ok = emqtt:disconnect(C3). + +t_disable_on_restart(_Config) -> + ok = emqx_eviction_agent:enable(test_eviction, undefined), + + ok = supervisor:terminate_child(emqx_eviction_agent_sup, emqx_eviction_agent), + {ok, _} = supervisor:restart_child(emqx_eviction_agent_sup, emqx_eviction_agent), + + ?assertEqual( + disabled, + emqx_eviction_agent:status() + ). + +t_session_serialization(_Config) -> + _ = erlang:process_flag(trap_exit, true), + ok = restart_emqx(), + + {ok, C0} = emqtt_connect(<<"client_with_session">>, false), + {ok, _, _} = emqtt:subscribe(C0, <<"t1">>), + ok = emqtt:disconnect(C0), + + ok = emqx_eviction_agent:enable(test_eviction, undefined), + + ?assertEqual( + 1, + emqx_eviction_agent:session_count() + ), + + %% Evacuate to the same node + + ?assertWaitEvent( + emqx_eviction_agent:evict_sessions(1, node()), + #{?snk_kind := emqx_channel_takeover_end, clientid := <<"client_with_session">>}, + 1000 + ), + + ok = emqx_eviction_agent:disable(test_eviction), + + ?assertEqual( + 1, + emqx_eviction_agent:session_count() + ), + + ?assertMatch( + #{data := [#{clientid := <<"client_with_session">>}]}, + emqx_mgmt_api:cluster_query( + emqx_channel_info, + #{}, + [], + fun emqx_mgmt_api_clients:qs2ms/2, + fun emqx_mgmt_api_clients:format_channel_info/2 + ) + ), + + mock_print(), + + ?assertPrinted( + "client_with_session", + emqx_mgmt_cli:clients(["list"]) + ), + + ?assertPrinted( + "client_with_session", + emqx_mgmt_cli:clients(["show", "client_with_session"]) + ), + + ?assertWaitEvent( + emqx_cm:kick_session(<<"client_with_session">>), + #{?snk_kind := emqx_cm_clean_down, client_id := <<"client_with_session">>}, + 1000 + ), + + ?assertEqual( + 0, + emqx_eviction_agent:session_count() + ). + +t_will_msg(_Config) -> + erlang:process_flag(trap_exit, true), + + WillMsg = <<"will_msg">>, + WillTopic = <<"will_topic">>, + ClientId = <<"client_with_will">>, + + _ = emqtt_connect([ + {clean_start, false}, + {clientid, ClientId}, + {will_payload, WillMsg}, + {will_topic, WillTopic} + ]), + + {ok, C} = emqtt_connect(), + {ok, _, _} = emqtt:subscribe(C, WillTopic), + + [ChanPid] = emqx_cm:lookup_channels(ClientId), + + ChanPid ! + {disconnect, ?RC_USE_ANOTHER_SERVER, use_another_server, #{ + 'Server-Reference' => <<>> + }}, + + receive + {publish, #{ + payload := WillMsg, + topic := WillTopic + }} -> + ok + after 1000 -> + ct:fail("Will message not received") + end, + + ok = emqtt:disconnect(C). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +% sn_connect_and_subscribe(ClientId, Topic) -> +% emqx_eviction_agent_test_helpers:sn_connect_and_subscribe(ClientId, Topic). + +assert_receive_publish([]) -> + ok; +assert_receive_publish([#{payload := Msg, topic := Topic} | Rest]) -> + receive + {publish, #{ + payload := Msg, + topic := Topic + }} -> + assert_receive_publish(Rest) + after 1000 -> + ?assert(false, "Message `" ++ binary_to_list(Msg) ++ "` is lost") + end. + +connect_and_publish(Topic, Message) -> + {ok, C} = emqtt_connect(), + emqtt:publish(C, Topic, Message), + ok = emqtt:disconnect(C). + +restart_emqx() -> + _ = application:stop(emqx), + _ = application:start(emqx), + _ = application:stop(emqx_eviction_agent), + _ = application:start(emqx_eviction_agent), + ok. + +mock_print() -> + catch meck:unload(emqx_ctl), + meck:new(emqx_ctl, [non_strict, passthrough]), + meck:expect(emqx_ctl, print, fun(Arg) -> emqx_ctl:format(Arg, []) end), + meck:expect(emqx_ctl, print, fun(Msg, Arg) -> emqx_ctl:format(Msg, Arg) end), + meck:expect(emqx_ctl, usage, fun(Usages) -> emqx_ctl:format_usage(Usages) end), + meck:expect(emqx_ctl, usage, fun(Cmd, Descr) -> emqx_ctl:format_usage(Cmd, Descr) end). diff --git a/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl b/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl new file mode 100644 index 000000000..3fe15e53a --- /dev/null +++ b/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl @@ -0,0 +1,69 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-import( + emqx_mgmt_api_test_util, + [ + request_api/2, + uri/1 + ] +). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_mgmt_api_test_util:init_suite([emqx_eviction_agent]), + Config. + +end_per_suite(Config) -> + emqx_mgmt_api_test_util:end_suite([emqx_eviction_agent]), + Config. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_status(_Config) -> + ?assertMatch( + {ok, #{<<"status">> := <<"disabled">>}}, + api_get(["node_eviction", "status"]) + ), + + ok = emqx_eviction_agent:enable(apitest, undefined), + + ?assertMatch( + {ok, #{ + <<"status">> := <<"enabled">>, + <<"stats">> := #{} + }}, + api_get(["node_eviction", "status"]) + ), + + ok = emqx_eviction_agent:disable(apitest), + + ?assertMatch( + {ok, #{<<"status">> := <<"disabled">>}}, + api_get(["node_eviction", "status"]) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +api_get(Path) -> + case request_api(get, uri(Path)) of + {ok, ResponseBody} -> + {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])}; + {error, _} = Error -> + Error + end. diff --git a/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl b/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl new file mode 100644 index 000000000..3b7ef6672 --- /dev/null +++ b/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl @@ -0,0 +1,251 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_channel_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). + +-define(CLIENT_ID, <<"client_with_session">>). + +-import( + emqx_eviction_agent_test_helpers, + [emqtt_connect/0, emqtt_connect/2] +). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_common_test_helpers:start_apps([emqx_conf, emqx_eviction_agent]), + {ok, _} = emqx:update_config([rpc, port_discovery], manual), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([emqx_eviction_agent, emqx_conf]). + +init_per_testcase(t_persistence, Config) -> + emqx_config:put([persistent_session_store, enabled], true), + {ok, _} = emqx_persistent_session_sup:start_link(), + emqx_persistent_session:init_db_backend(), + ?assert(emqx_persistent_session:is_store_enabled()), + Config; +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(t_persistence, Config) -> + emqx_config:put([persistent_session_store, enabled], false), + emqx_persistent_session:init_db_backend(), + ?assertNot(emqx_persistent_session:is_store_enabled()), + Config; +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_start_no_session(_Config) -> + Opts = #{ + clientinfo => #{ + clientid => ?CLIENT_ID, + zone => internal + }, + conninfo => #{ + clientid => ?CLIENT_ID, + receive_maximum => 32, + expiry_interval => 10000 + } + }, + ?assertMatch( + {error, {no_session, _}}, + emqx_eviction_agent_channel:start_supervised(Opts) + ). + +t_start_no_expire(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + + Opts = #{ + clientinfo => #{ + clientid => ?CLIENT_ID, + zone => internal + }, + conninfo => #{ + clientid => ?CLIENT_ID, + receive_maximum => 32, + expiry_interval => 0 + } + }, + ?assertMatch( + {error, {should_be_expired, _}}, + emqx_eviction_agent_channel:start_supervised(Opts) + ). + +t_start_infinite_expire(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + + Opts = #{ + clientinfo => #{ + clientid => ?CLIENT_ID, + zone => internal + }, + conninfo => #{ + clientid => ?CLIENT_ID, + receive_maximum => 32, + expiry_interval => ?UINT_MAX + } + }, + ?assertMatch( + {ok, _}, + emqx_eviction_agent_channel:start_supervised(Opts) + ). + +t_kick(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + Opts = evict_session_opts(?CLIENT_ID), + + {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts), + + ?assertEqual( + ok, + emqx_eviction_agent_channel:call(Pid, kick) + ). + +t_discard(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + Opts = evict_session_opts(?CLIENT_ID), + + {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts), + + ?assertEqual( + ok, + emqx_eviction_agent_channel:call(Pid, discard) + ). + +t_stop(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + Opts = evict_session_opts(?CLIENT_ID), + + {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts), + + ?assertEqual( + ok, + emqx_eviction_agent_channel:stop(Pid) + ). + +t_ignored_calls(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + Opts = evict_session_opts(?CLIENT_ID), + + {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts), + + ok = emqx_eviction_agent_channel:cast(Pid, unknown), + Pid ! unknown, + + ?assertEqual( + [], + emqx_eviction_agent_channel:call(Pid, list_acl_cache) + ), + + ?assertEqual( + ok, + emqx_eviction_agent_channel:call(Pid, {quota, quota}) + ), + + ?assertEqual( + ignored, + emqx_eviction_agent_channel:call(Pid, unknown) + ). + +t_expire(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + #{conninfo := ConnInfo} = Opts0 = evict_session_opts(?CLIENT_ID), + Opts1 = Opts0#{conninfo => ConnInfo#{expiry_interval => 1}}, + + {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts1), + + ct:sleep(1500), + + ?assertNot(is_process_alive(Pid)). + +t_get_connected_client_count(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + + ?assertEqual( + 1, + emqx_cm:get_connected_client_count() + ), + + Opts = evict_session_opts(?CLIENT_ID), + + {ok, _} = emqx_eviction_agent_channel:start_supervised(Opts), + + ?assertEqual( + 0, + emqx_cm:get_connected_client_count() + ). + +t_persistence(_Config) -> + erlang:process_flag(trap_exit, true), + + Topic = <<"t1">>, + Message = <<"message_to_persist">>, + + {ok, C0} = emqtt_connect(?CLIENT_ID, false), + {ok, _, _} = emqtt:subscribe(C0, Topic, 0), + + Opts = evict_session_opts(?CLIENT_ID), + {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts), + + {ok, C1} = emqtt_connect(), + {ok, _} = emqtt:publish(C1, Topic, Message, 1), + ok = emqtt:disconnect(C1), + + %% Kill channel so that the session is only persisted + ok = emqx_eviction_agent_channel:call(Pid, kick), + + %% Should restore session from persistents storage and receive messages + {ok, C2} = emqtt_connect(?CLIENT_ID, false), + + receive + {publish, #{ + payload := Message, + topic := Topic + }} -> + ok + after 1000 -> + ct:fail("message not received") + end, + + ok = emqtt:disconnect(C2). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +evict_session_opts(ClientId) -> + maps:with( + [conninfo, clientinfo], + emqx_cm:get_chan_info(ClientId) + ). diff --git a/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl b/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl new file mode 100644 index 000000000..4cfb2fff5 --- /dev/null +++ b/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl @@ -0,0 +1,39 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_cli_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_common_test_helpers:start_apps([emqx_eviction_agent]), + Config. + +end_per_suite(Config) -> + _ = emqx_eviction_agent:disable(foo), + emqx_common_test_helpers:stop_apps([emqx_eviction_agent]), + Config. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_status(_Config) -> + %% usage + ok = emqx_eviction_agent_cli:cli(["foobar"]), + + %% status + ok = emqx_eviction_agent_cli:cli(["status"]), + + ok = emqx_eviction_agent:enable(foo, undefined), + + %% status + ok = emqx_eviction_agent_cli:cli(["status"]). diff --git a/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl b/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl new file mode 100644 index 000000000..8f88ebf97 --- /dev/null +++ b/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl @@ -0,0 +1,141 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_test_helpers). + +-export([ + emqtt_connect/0, + emqtt_connect/1, + emqtt_connect/2, + emqtt_connect_many/2, + stop_many/1, + + emqtt_try_connect/1, + + start_cluster/2, + start_cluster/3, + stop_cluster/2, + + case_specific_node_name/2, + case_specific_node_name/3, + concat_atoms/1 +]). + +emqtt_connect() -> + emqtt_connect(<<"client1">>, true). + +emqtt_connect(ClientId, CleanStart) -> + emqtt_connect([{clientid, ClientId}, {clean_start, CleanStart}]). + +emqtt_connect(Opts) -> + {ok, C} = emqtt:start_link( + Opts ++ + [ + {proto_ver, v5}, + {properties, #{'Session-Expiry-Interval' => 600}} + ] + ), + case emqtt:connect(C) of + {ok, _} -> {ok, C}; + {error, _} = Error -> Error + end. + +emqtt_connect_many(Port, Count) -> + lists:map( + fun(N) -> + NBin = integer_to_binary(N), + ClientId = <<"client-", NBin/binary>>, + {ok, C} = emqtt_connect([{clientid, ClientId}, {clean_start, false}, {port, Port}]), + C + end, + lists:seq(1, Count) + ). + +stop_many(Clients) -> + lists:foreach( + fun(C) -> + catch emqtt:disconnect(C) + end, + Clients + ), + ct:sleep(100). + +emqtt_try_connect(Opts) -> + case emqtt_connect(Opts) of + {ok, C} -> + emqtt:disconnect(C), + ok; + {error, _} = Error -> + Error + end. + +start_cluster(NamesWithPorts, Apps) -> + start_cluster(NamesWithPorts, Apps, []). + +start_cluster(NamesWithPorts, Apps, Env) -> + Specs = lists:map( + fun({ShortName, Port}) -> + {core, ShortName, #{listener_ports => [{tcp, Port}]}} + end, + NamesWithPorts + ), + Opts0 = [ + {env, [{emqx, boot_modules, [broker, listeners]}]}, + {apps, Apps}, + {conf, + [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]] ++ + [{[rpc, mode], async}]}, + {env, Env} + ], + Cluster = emqx_common_test_helpers:emqx_cluster( + Specs, + Opts0 + ), + NodesWithPorts = [ + { + emqx_common_test_helpers:start_slave(Name, Opts), + proplists:get_value(Name, NamesWithPorts) + } + || {Name, Opts} <- Cluster + ], + ok = lists:foreach( + fun({Node, _Port}) -> + snabbkaffe:forward_trace(Node) + end, + NodesWithPorts + ), + NodesWithPorts. + +stop_cluster(NodesWithPorts, Apps) -> + lists:foreach( + fun({Node, _Port}) -> + lists:foreach( + fun(App) -> + rpc:call(Node, application, stop, [App]) + end, + Apps + ), + %% This sleep is just to make logs cleaner + ct:sleep(100), + _ = rpc:call(Node, emqx_common_test_helpers, stop_apps, []), + emqx_common_test_helpers:stop_slave(Node) + end, + NodesWithPorts + ). + +case_specific_node_name(Module, Case) -> + concat_atoms([Module, '__', Case]). + +case_specific_node_name(Module, Case, Node) -> + concat_atoms([Module, '__', Case, '__', Node]). + +concat_atoms(Atoms) -> + binary_to_atom( + iolist_to_binary( + lists:map( + fun atom_to_binary/1, + Atoms + ) + ) + ). diff --git a/lib-ee/emqx_node_rebalance/README.md b/lib-ee/emqx_node_rebalance/README.md new file mode 100644 index 000000000..2e56f62cd --- /dev/null +++ b/lib-ee/emqx_node_rebalance/README.md @@ -0,0 +1,9 @@ +emqx_node_rebalance +===== + +An OTP library + +Build +----- + + $ rebar3 compile diff --git a/lib-ee/emqx_node_rebalance/etc/emqx_node_rebalance.conf b/lib-ee/emqx_node_rebalance/etc/emqx_node_rebalance.conf new file mode 100644 index 000000000..8ace22435 --- /dev/null +++ b/lib-ee/emqx_node_rebalance/etc/emqx_node_rebalance.conf @@ -0,0 +1,3 @@ +##-------------------------------------------------------------------- +## EMQX Node Rebalance Plugin +##-------------------------------------------------------------------- diff --git a/lib-ee/emqx_node_rebalance/i18n/emqx_node_rebalance_api_i18n.conf b/lib-ee/emqx_node_rebalance/i18n/emqx_node_rebalance_api_i18n.conf new file mode 100644 index 000000000..f5f161a92 --- /dev/null +++ b/lib-ee/emqx_node_rebalance/i18n/emqx_node_rebalance_api_i18n.conf @@ -0,0 +1,490 @@ +emqx_node_rebalance_api { + + ## API Request Fields + + load_rebalance_status { + desc { + en: "Get rebalance status of the current node" + zh: "获取当前节点的rebalance状态" + } + label { + en: "Get rebalance status" + zh: "获取rebalance状态" + } + } + + load_rebalance_global_status { + desc { + en: "Get status of all rebalance/evacuation processes across the cluster" + zh: "获取集群中所有rebalance/evacuation进程的状态" + } + label { + en: "Get global rebalance status" + zh: "获取全局rebalance状态" + } + } + + load_rebalance_availability_check { + desc { + en: "Check if the node is being evacuated or rebalanced" + zh: "检查节点是否正在被evacuate或rebalance" + } + label { + en: "Availability check" + zh: "可用性检查" + } + } + + load_rebalance_start { + desc { + en: "Start rebalance process" + zh: "启动rebalance进程" + } + label { + en: "Start rebalance" + zh: "启动rebalance" + } + } + + load_rebalance_stop { + desc { + en: "Stop rebalance process" + zh: "停止rebalance进程" + } + label { + en: "Stop rebalance" + zh: "停止rebalance" + } + } + + load_rebalance_evacuation_start { + desc { + en: "Start evacuation process" + zh: "启动evacuation进程" + } + label { + en: "Start evacuation" + zh: "启动evacuation" + } + } + + load_rebalance_evacuation_stop { + desc { + en: "Stop evacuation process" + zh: "停止evacuation进程" + } + label { + en: "Stop evacuation" + zh: "停止evacuation" + } + } + + param_node { + desc { + en: "Node name" + zh: "节点名称" + } + label { + en: "Node name" + zh: "节点名称" + } + } + + wait_health_check { + desc { + en: "Time to wait before starting the rebalance process, in seconds" + zh: "启动rebalance进程前等待的时间，单位为秒" + } + label { + en: "Wait health check" + zh: "等待健康检查" + } + } + + conn_evict_rate { + desc { + en: "The rate of evicting connections, in connections per second" + zh: "逐出连接的速率，以每秒连接数表示" + } + label { + en: "Connection eviction rate" + zh: "连接驱逐率" + } + } + + sess_evict_rate { + desc { + en: "The rate of evicting sessions, in sessions per second" + zh: "逐出会话的速率，以每秒会话为单位" + } + label { + en: "Session eviction rate" + zh: "会话驱逐率" + } + } + + abs_conn_threshold { + desc { + en: "Maximum desired difference between the number of connections on the node and the average number of connections on the recipient nodes" + zh: "节点上的连接数与接收节点上的平均连接数之间的最大期望差值" + } + label { + en: "Absolute connection threshold" + zh: "绝对连接阈值" + } + } + + rel_conn_threshold { + desc { + en: "Maximum desired fraction between the number of connections on the node and the average number of connections on the recipient nodes" + zh: "节点上的连接数与接收节点上的平均连接数之间的最大期望分数" + } + label { + en: "Relative connection threshold" + zh: "相对连接阈值" + } + } + + abs_sess_threshold { + desc { + en: "Maximum desired difference between the number of sessions on the node and the average number of sessions on the recipient nodes" + zh: "节点上的会话数与接收节点上的平均会话数之间的最大期望差异" + } + label { + en: "Absolute session threshold" + zh: "绝对会话阈值" + } + } + + rel_sess_threshold { + desc { + en: "Maximum desired fraction between the number of sessions on the node and the average number of sessions on the recipient nodes" + zh: "节点上的会话数与接收节点上的平均会话数之间的最大期望分数" + } + label { + en: "Relative session threshold" + zh: "相对会话阈值" + } + } + + wait_takeover { + desc { + en: "Time to wait before starting session evacuation process, in seconds" + zh: "开始会话疏散过程之前等待的时间，以秒为单位" + } + label { + en: "Wait takeover" + zh: "等待接管" + } + } + + redirect_to { + desc { + en: "Server reference to redirect clients to (MQTTv5 Server redirection)" + zh: "将客户端重定向到的服务器参考（MQTTv5 服务器重定向）" + } + label { + en: "Redirect to" + zh: "重定向至" + } + } + + migrate_to { + desc { + en: "Nodes to migrate sessions to" + zh: "将会话迁移到的节点" + } + label { + en: "Migrate to" + zh: "迁移到" + } + } + + rebalance_nodes { + desc { + en: "Nodes to participate in rebalance" + zh: "参与rebalance的节点" + } + label { + en: "Rebalance nodes" + zh: "重新平衡节点" + } + } + + ## API Response Fields + + local_status_enabled { + desc { + en: "Whether the node is being evacuated" + zh: "节点是否正在撤离" + } + label { + en: "Local evacuation status" + zh: "当地避难状况" + } + } + + local_status_process { + desc { + en: "The process that is being performed on the node: evacuation or rebalance" + zh: "正在节点上执行的过程：疏散或重新平衡" + } + label { + en: "Node process" + zh: "节点进程" + } + } + + local_status_state { + desc { + en: "The state of the process that is being performed on the node" + zh: "正在节点上执行的进程的状态" + } + label { + en: "Rebalance/evacuation current state" + zh: "重新平衡/疏散当前状态" + } + } + + local_status_coordinator_node { + desc { + en: "The node that is coordinating rebalance process" + zh: "协调再平衡过程的节点" + } + label { + en: "Coordinator node" + zh: "协调节点" + } + } + + local_status_connection_eviction_rate { + desc { + en: "The rate of evicting connections, in connections per second" + zh: "逐出连接的速率，以每秒连接数表示" + } + label { + en: "Connection eviction rate" + zh: "连接驱逐率" + } + } + + local_status_session_eviction_rate { + desc { + en: "The rate of evicting sessions, in sessions per second" + zh: "逐出会话的速率，以每秒会话为单位" + } + label { + en: "Session eviction rate" + zh: "会话驱逐率" + } + } + + local_status_connection_goal { + desc { + en: "The number of connections that the node should have after the rebalance/evacuation process" + zh: "节点在重新平衡/疏散过程后应该拥有的连接数" + } + label { + en: "Connection goal" + zh: "连接目标" + } + } + + local_status_session_goal { + desc { + en: "The number of sessions that the node should have after the evacuation process" + zh: "疏散过程后节点应有的会话数" + } + label { + en: "Session goal" + zh: "会话目标" + } + } + + local_status_disconnected_session_goal { + desc { + en: "The number of disconnected sessions that the node should have after the rebalance process" + zh: "重新平衡过程后节点应具有的断开连接的会话数" + } + label { + en: "Disconnected session goal" + zh: "断开连接的会话目标" + } + } + + local_status_session_recipients { + desc { + en: "List of nodes to which sessions are being evacuated" + zh: "会话被疏散到的节点列表" + } + label { + en: "Session recipients" + zh: "会话收件人" + } + } + + local_status_recipients { + desc { + en: "List of nodes to which connections/sessions are being evacuated during rebalance" + zh: "在重新平衡期间连接/会话被疏散到的节点列表" + } + label { + en: "Recipients" + zh: "收件人" + } + } + + local_status_stats { + desc { + en: "Statistics of the evacuation/rebalance process" + zh: "疏散/再平衡过程的统计" + } + label { + en: "Statistics" + zh: "统计数据" + } + } + + status_stats_initial_connected { + desc { + en: "The number of connections on the node before the evacuation/rebalance process" + zh: "疏散/重新平衡过程之前节点上的连接数" + } + label { + en: "Initial connected" + zh: "初始连接" + } + } + + status_stats_current_connected { + desc { + en: "Current number of connections on the node" + zh: "节点上的当前连接数" + } + label { + en: "Current connections" + zh: "当前连接" + } + } + + status_stats_initial_sessions { + desc { + en: "The number of sessions on the node before the evacuation/rebalance process" + zh: "疏散/重新平衡过程之前节点上的会话数" + } + label { + en: "Initial sessions" + zh: "初始会话" + } + } + + status_stats_current_sessions { + desc { + en: "Current number of sessions on the node" + zh: "节点上的当前会话数" + } + label { + en: "Current sessions" + zh: "当前会话" + } + } + + status_stats_current_disconnected_sessions { + desc { + en: "Current number of disconnected sessions on the node" + zh: "节点上当前断开连接的会话数" + } + label { + en: "Current disconnected sessions" + zh: "当前断开连接的会话" + } + } + + coordinator_status_donors { + desc { + en: "List of nodes from which connections/sessions are being evacuated" + zh: "正在疏散连接/会话的节点列表" + } + label { + en: "Donors" + zh: "捐助者" + } + } + + coordinator_status_donor_conn_avg { + desc { + en: "Average number of connections per donor node" + zh: "每个供体节点的平均连接数" + } + label { + en: "Donor connections average" + zh: "捐助者连接平均值" + } + } + + coordinator_status_donor_sess_avg { + desc { + en: "Average number of sessions per donor node" + zh: "每个供体节点的平均会话数" + } + label { + en: "Donor sessions average" + zh: "平均捐助会议" + } + } + + coordinator_status_node { + desc { + en: "The node that is coordinating the evacuation/rebalance process" + zh: "协调疏散/再平衡过程的节点" + } + label { + en: "Coordinator node" + zh: "协调节点" + } + } + + evacuation_status_node { + desc { + en: "The node that is being evacuated" + zh: "正在撤离的节点" + } + label { + en: "Evacuated node" + zh: "疏散节点" + } + } + + global_status_evacuations { + desc { + en: "List of nodes that are being evacuated" + zh: "正在撤离的节点列表" + } + label { + en: "Evacuations" + zh: "疏散" + } + } + + global_status_rebalances { + desc { + en: "List of nodes that coordinate a rebalance" + zh: "协调再平衡的节点列表" + } + label { + en: "Rebalances" + zh: "再平衡" + } + } + + empty_response { + desc { + en: "The response is empty" + zh: "响应为空" + } + label { + en: "Empty response" + zh: "空响应" + } + } +} diff --git a/lib-ee/emqx_node_rebalance/include/emqx_node_rebalance.hrl b/lib-ee/emqx_node_rebalance/include/emqx_node_rebalance.hrl new file mode 100644 index 000000000..ccc671e81 --- /dev/null +++ b/lib-ee/emqx_node_rebalance/include/emqx_node_rebalance.hrl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-define(DEFAULT_CONN_EVICT_RATE, 500). +-define(DEFAULT_SESS_EVICT_RATE, 500). + +%% sec +-define(DEFAULT_WAIT_HEALTH_CHECK, 60). +%% sec +-define(DEFAULT_WAIT_TAKEOVER, 60). + +-define(DEFAULT_ABS_CONN_THRESHOLD, 1000). +-define(DEFAULT_ABS_SESS_THRESHOLD, 1000). + +-define(DEFAULT_REL_CONN_THRESHOLD, 1.1). +-define(DEFAULT_REL_SESS_THRESHOLD, 1.1). + +-define(EVICT_INTERVAL, 1000). + +-define(EVACUATION_FILENAME, <<".evacuation">>). diff --git a/lib-ee/emqx_node_rebalance/rebar.config b/lib-ee/emqx_node_rebalance/rebar.config new file mode 100644 index 000000000..b055d8f4f --- /dev/null +++ b/lib-ee/emqx_node_rebalance/rebar.config @@ -0,0 +1,2 @@ +{deps, [{emqx, {path, "../../apps/emqx"}}]}. +{project_plugins, [erlfmt]}. diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.app.src b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.app.src new file mode 100644 index 000000000..9673e4fda --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.app.src @@ -0,0 +1,22 @@ +{application, emqx_node_rebalance, [ + {description, "EMQX Node Rebalance"}, + {vsn, "5.0.0"}, + {registered, [ + emqx_node_rebalance_sup, + emqx_node_rebalance, + emqx_node_rebalance_agent, + emqx_node_rebalance_evacuation + ]}, + {applications, [ + kernel, + stdlib + ]}, + {mod, {emqx_node_rebalance_app, []}}, + {env, []}, + {modules, []}, + {maintainers, ["EMQX Team "]}, + {links, [ + {"Homepage", "https://emqx.io/"}, + {"Github", "https://github.com/emqx"} + ]} +]}. diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.appup.src b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.appup.src new file mode 100644 index 000000000..c1b84778d --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.appup.src @@ -0,0 +1,3 @@ +%% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! +{VSN, [{<<".*">>, []}], [{<<".*">>, []}]}. diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.erl b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.erl new file mode 100644 index 000000000..1f2adc565 --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.erl @@ -0,0 +1,438 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance). + +-include("emqx_node_rebalance.hrl"). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-export([ + start/1, + status/0, + status/1, + stop/0 +]). + +-export([start_link/0]). + +-behaviour(gen_statem). + +-export([ + init/1, + callback_mode/0, + handle_event/4, + code_change/4 +]). + +-export([ + is_node_available/0, + available_nodes/1, + connection_count/0, + session_count/0, + disconnected_session_count/0 +]). + +-export_type([ + start_opts/0, + start_error/0 +]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +-type start_opts() :: #{ + conn_evict_rate => pos_integer(), + sess_evict_rate => pos_integer(), + wait_health_check => pos_integer(), + wait_takeover => pos_integer(), + abs_conn_threshold => pos_integer(), + rel_conn_threshold => number(), + abs_sess_threshold => pos_integer(), + rel_sess_threshold => number(), + nodes => [node()] +}. +-type start_error() :: already_started | [{node(), term()}]. + +-spec start(start_opts()) -> ok_or_error(start_error()). +start(StartOpts) -> + Opts = maps:merge(default_opts(), StartOpts), + gen_statem:call(?MODULE, {start, Opts}). + +-spec stop() -> ok_or_error(not_started). +stop() -> + gen_statem:call(?MODULE, stop). + +-spec status() -> disabled | {enabled, map()}. +status() -> + gen_statem:call(?MODULE, status). + +-spec status(pid()) -> disabled | {enabled, map()}. +status(Pid) -> + gen_statem:call(Pid, status). + +-spec start_link() -> startlink_ret(). +start_link() -> + gen_statem:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec available_nodes(list(node())) -> list(node()). +available_nodes(Nodes) when is_list(Nodes) -> + {Available, _} = emqx_node_rebalance_proto_v1:available_nodes(Nodes), + lists:filter(fun is_atom/1, Available). + +%%-------------------------------------------------------------------- +%% gen_statem callbacks +%%-------------------------------------------------------------------- + +callback_mode() -> handle_event_function. + +%% states: disabled, wait_health_check, evicting_conns, wait_takeover, evicting_sessions + +init([]) -> + ?tp(debug, emqx_node_rebalance_started, #{}), + {ok, disabled, #{}}. + +%% start +handle_event( + {call, From}, + {start, #{wait_health_check := WaitHealthCheck} = Opts}, + disabled, + #{} = Data +) -> + case enable_rebalance(Data#{opts => Opts}) of + {ok, NewData} -> + ?SLOG(warning, #{msg => "node_rebalance_enabled", opts => Opts}), + {next_state, wait_health_check, NewData, [ + {state_timeout, seconds(WaitHealthCheck), evict_conns}, + {reply, From, ok} + ]}; + {error, Reason} -> + ?SLOG(warning, #{ + msg => "node_rebalance_enable_failed", + reason => Reason + }), + {keep_state_and_data, [{reply, From, {error, Reason}}]} + end; +handle_event({call, From}, {start, _Opts}, _State, #{}) -> + {keep_state_and_data, [{reply, From, {error, already_started}}]}; +%% stop +handle_event({call, From}, stop, disabled, #{}) -> + {keep_state_and_data, [{reply, From, {error, not_started}}]}; +handle_event({call, From}, stop, _State, Data) -> + ok = disable_rebalance(Data), + ?SLOG(warning, #{msg => "node_rebalance_stopped"}), + {next_state, disabled, deinit(Data), [{reply, From, ok}]}; +%% status +handle_event({call, From}, status, disabled, #{}) -> + {keep_state_and_data, [{reply, From, disabled}]}; +handle_event({call, From}, status, State, Data) -> + Stats = get_stats(State, Data), + {keep_state_and_data, [ + {reply, From, + {enabled, Stats#{ + state => State, + coordinator_node => node() + }}} + ]}; +%% conn eviction +handle_event( + state_timeout, + evict_conns, + wait_health_check, + Data +) -> + ?SLOG(warning, #{msg => "node_rebalance_wait_health_check_over"}), + {next_state, evicting_conns, Data, [{state_timeout, 0, evict_conns}]}; +handle_event( + state_timeout, + evict_conns, + evicting_conns, + #{ + opts := #{ + wait_takeover := WaitTakeover, + evict_interval := EvictInterval + } + } = Data +) -> + case evict_conns(Data) of + ok -> + ?SLOG(warning, #{msg => "node_rebalance_evict_conns_over"}), + {next_state, wait_takeover, Data, [ + {state_timeout, seconds(WaitTakeover), evict_sessions} + ]}; + {continue, NewData} -> + {keep_state, NewData, [{state_timeout, EvictInterval, evict_conns}]} + end; +handle_event( + state_timeout, + evict_sessions, + wait_takeover, + Data +) -> + ?SLOG(warning, #{msg => "node_rebalance_wait_takeover_over"}), + {next_state, evicting_sessions, Data, [{state_timeout, 0, evict_sessions}]}; +handle_event( + state_timeout, + evict_sessions, + evicting_sessions, + #{opts := #{evict_interval := EvictInterval}} = Data +) -> + case evict_sessions(Data) of + ok -> + ?tp(debug, emqx_node_rebalance_evict_sess_over, #{}), + ?SLOG(warning, #{msg => "node_rebalance_evict_sessions_over"}), + ok = disable_rebalance(Data), + ?SLOG(warning, #{msg => "node_rebalance_finished_successfully"}), + {next_state, disabled, deinit(Data)}; + {continue, NewData} -> + {keep_state, NewData, [{state_timeout, EvictInterval, evict_sessions}]} + end; +handle_event({call, From}, Msg, _State, _Data) -> + ?SLOG(warning, #{msg => "node_rebalance_unknown_call", call => Msg}), + {keep_state_and_data, [{reply, From, ignored}]}; +handle_event(info, Msg, _State, _Data) -> + ?SLOG(warning, #{msg => "node_rebalance_unknown_info", info => Msg}), + keep_state_and_data; +handle_event(cast, Msg, _State, _Data) -> + ?SLOG(warning, #{msg => "node_rebalance_unknown_cast", cast => Msg}), + keep_state_and_data. + +code_change(_Vsn, State, Data, _Extra) -> + {ok, State, Data}. + +%%-------------------------------------------------------------------- +%% internal funs +%%-------------------------------------------------------------------- + +enable_rebalance(#{opts := Opts} = Data) -> + Nodes = maps:get(nodes, Opts), + ConnCounts = multicall(Nodes, connection_counts, []), + SessCounts = multicall(Nodes, session_counts, []), + {_, Counts} = lists:unzip(ConnCounts), + Avg = avg(Counts), + {DonorCounts, RecipientCounts} = lists:partition( + fun({_Node, Count}) -> + Count >= Avg + end, + ConnCounts + ), + ?SLOG(warning, #{ + msg => "node_rebalance_enabling", + conn_counts => ConnCounts, + donor_counts => DonorCounts, + recipient_counts => RecipientCounts + }), + {DonorNodes, _} = lists:unzip(DonorCounts), + {RecipientNodes, _} = lists:unzip(RecipientCounts), + case need_rebalance(DonorNodes, RecipientNodes, ConnCounts, SessCounts, Opts) of + false -> + {error, nothing_to_balance}; + true -> + _ = multicall(DonorNodes, enable_rebalance_agent, [self()]), + {ok, Data#{ + donors => DonorNodes, + recipients => RecipientNodes, + initial_conn_counts => maps:from_list(ConnCounts), + initial_sess_counts => maps:from_list(SessCounts) + }} + end. + +disable_rebalance(#{donors := DonorNodes}) -> + _ = multicall(DonorNodes, disable_rebalance_agent, [self()]), + ok. + +evict_conns(#{donors := DonorNodes, recipients := RecipientNodes, opts := Opts} = Data) -> + DonorNodeCounts = multicall(DonorNodes, connection_counts, []), + {_, DonorCounts} = lists:unzip(DonorNodeCounts), + RecipientNodeCounts = multicall(RecipientNodes, connection_counts, []), + {_, RecipientCounts} = lists:unzip(RecipientNodeCounts), + + DonorAvg = avg(DonorCounts), + RecipientAvg = avg(RecipientCounts), + Thresholds = thresholds(conn, Opts), + NewData = Data#{ + donor_conn_avg => DonorAvg, + recipient_conn_avg => RecipientAvg, + donor_conn_counts => maps:from_list(DonorNodeCounts), + recipient_conn_counts => maps:from_list(RecipientNodeCounts) + }, + case within_thresholds(DonorAvg, RecipientAvg, Thresholds) of + true -> + ok; + false -> + ConnEvictRate = maps:get(conn_evict_rate, Opts), + NodesToEvict = nodes_to_evict(RecipientAvg, DonorNodeCounts), + ?SLOG(warning, #{ + msg => "node_rebalance_evict_conns", + nodes => NodesToEvict, + counts => ConnEvictRate + }), + _ = multicall(NodesToEvict, evict_connections, [ConnEvictRate]), + {continue, NewData} + end. + +evict_sessions(#{donors := DonorNodes, recipients := RecipientNodes, opts := Opts} = Data) -> + DonorNodeCounts = multicall(DonorNodes, disconnected_session_counts, []), + {_, DonorCounts} = lists:unzip(DonorNodeCounts), + RecipientNodeCounts = multicall(RecipientNodes, disconnected_session_counts, []), + {_, RecipientCounts} = lists:unzip(RecipientNodeCounts), + + DonorAvg = avg(DonorCounts), + RecipientAvg = avg(RecipientCounts), + Thresholds = thresholds(sess, Opts), + NewData = Data#{ + donor_sess_avg => DonorAvg, + recipient_sess_avg => RecipientAvg, + donor_sess_counts => maps:from_list(DonorNodeCounts), + recipient_sess_counts => maps:from_list(RecipientNodeCounts) + }, + case within_thresholds(DonorAvg, RecipientAvg, Thresholds) of + true -> + ok; + false -> + SessEvictRate = maps:get(sess_evict_rate, Opts), + NodesToEvict = nodes_to_evict(RecipientAvg, DonorNodeCounts), + ?SLOG(warning, #{ + msg => "node_rebalance_evict_sessions", + nodes => NodesToEvict, + counts => SessEvictRate + }), + _ = multicall( + NodesToEvict, + evict_sessions, + [SessEvictRate, RecipientNodes, disconnected] + ), + {continue, NewData} + end. + +need_rebalance([] = _DonorNodes, _RecipientNodes, _ConnCounts, _SessCounts, _Opts) -> + false; +need_rebalance(_DonorNodes, [] = _RecipientNodes, _ConnCounts, _SessCounts, _Opts) -> + false; +need_rebalance(DonorNodes, RecipientNodes, ConnCounts, SessCounts, Opts) -> + DonorConnAvg = avg_for_nodes(DonorNodes, ConnCounts), + RecipientConnAvg = avg_for_nodes(RecipientNodes, ConnCounts), + DonorSessAvg = avg_for_nodes(DonorNodes, SessCounts), + RecipientSessAvg = avg_for_nodes(RecipientNodes, SessCounts), + Result = + (not within_thresholds(DonorConnAvg, RecipientConnAvg, thresholds(conn, Opts))) orelse + (not within_thresholds(DonorSessAvg, RecipientSessAvg, thresholds(sess, Opts))), + ?tp( + debug, + emqx_node_rebalance_need_rebalance, + #{ + donors => DonorNodes, + recipients => RecipientNodes, + conn_counts => ConnCounts, + sess_counts => SessCounts, + opts => Opts, + result => Result + } + ), + Result. + +avg_for_nodes(Nodes, Counts) -> + avg(maps:values(maps:with(Nodes, maps:from_list(Counts)))). + +within_thresholds(Value, GoalValue, {AbsThres, RelThres}) -> + (Value =< GoalValue + AbsThres) orelse (Value =< GoalValue * RelThres). + +thresholds(conn, #{abs_conn_threshold := Abs, rel_conn_threshold := Rel}) -> + {Abs, Rel}; +thresholds(sess, #{abs_sess_threshold := Abs, rel_sess_threshold := Rel}) -> + {Abs, Rel}. + +nodes_to_evict(Goal, NodeCounts) -> + {Nodes, _} = lists:unzip( + lists:filter( + fun({_Node, Count}) -> + Count > Goal + end, + NodeCounts + ) + ), + Nodes. + +get_stats(disabled, _Data) -> #{}; +get_stats(_State, Data) -> Data. + +avg(List) when length(List) >= 1 -> + lists:sum(List) / length(List). + +multicall(Nodes, F, A) -> + case apply(emqx_node_rebalance_proto_v1, F, [Nodes | A]) of + {Results, []} -> + case lists:partition(fun is_ok/1, lists:zip(Nodes, Results)) of + {OkResults, []} -> + [{Node, ok_result(Result)} || {Node, Result} <- OkResults]; + {_, BadResults} -> + error({bad_nodes, BadResults}) + end; + {_, [_BadNode | _] = BadNodes} -> + error({bad_nodes, BadNodes}) + end. + +is_ok({_Node, {ok, _}}) -> true; +is_ok({_Node, ok}) -> true; +is_ok(_) -> false. + +ok_result({ok, Result}) -> Result; +ok_result(ok) -> ok. + +connection_count() -> + {ok, emqx_eviction_agent:connection_count()}. + +session_count() -> + {ok, emqx_eviction_agent:session_count()}. + +disconnected_session_count() -> + {ok, emqx_eviction_agent:session_count(disconnected)}. + +default_opts() -> + #{ + conn_evict_rate => ?DEFAULT_CONN_EVICT_RATE, + abs_conn_threshold => ?DEFAULT_ABS_CONN_THRESHOLD, + rel_conn_threshold => ?DEFAULT_REL_CONN_THRESHOLD, + + sess_evict_rate => ?DEFAULT_SESS_EVICT_RATE, + abs_sess_threshold => ?DEFAULT_ABS_SESS_THRESHOLD, + rel_sess_threshold => ?DEFAULT_REL_SESS_THRESHOLD, + + wait_health_check => ?DEFAULT_WAIT_HEALTH_CHECK, + wait_takeover => ?DEFAULT_WAIT_TAKEOVER, + + evict_interval => ?EVICT_INTERVAL, + + nodes => all_nodes() + }. + +deinit(Data) -> + Keys = [ + recipient_conn_avg, + recipient_sess_avg, + donor_conn_avg, + donor_sess_avg, + recipient_conn_counts, + recipient_sess_counts, + donor_conn_counts, + donor_sess_counts, + initial_conn_counts, + initial_sess_counts, + opts + ], + maps:without(Keys, Data). + +is_node_available() -> + true = is_pid(whereis(emqx_node_rebalance_agent)), + disabled = emqx_eviction_agent:status(), + node(). + +all_nodes() -> + mria_mnesia:running_nodes(). + +seconds(Sec) -> + round(timer:seconds(Sec)). diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl new file mode 100644 index 000000000..47708d00e --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl @@ -0,0 +1,131 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_agent). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). + +-include_lib("stdlib/include/qlc.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-export([ + start_link/0, + enable/1, + disable/1, + status/0 +]). + +-export([ + init/1, + handle_call/3, + handle_info/2, + handle_cast/2, + code_change/3 +]). + +-define(ENABLE_KIND, emqx_node_rebalance). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +-type status() :: {enabled, pid()} | disabled. + +-spec start_link() -> startlink_ret(). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec enable(pid()) -> ok_or_error(already_enabled | eviction_agent_busy). +enable(CoordinatorPid) -> + gen_server:call(?MODULE, {enable, CoordinatorPid}). + +-spec disable(pid()) -> ok_or_error(already_disabled | invalid_coordinator). +disable(CoordinatorPid) -> + gen_server:call(?MODULE, {disable, CoordinatorPid}). + +-spec status() -> status(). +status() -> + gen_server:call(?MODULE, status). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([]) -> + {ok, #{}}. + +handle_call({enable, CoordinatorPid}, _From, St) -> + case St of + #{coordinator_pid := _Pid} -> + {reply, {error, already_enabled}, St}; + _ -> + true = link(CoordinatorPid), + EvictionAgentPid = whereis(emqx_eviction_agent), + true = link(EvictionAgentPid), + case emqx_eviction_agent:enable(?ENABLE_KIND, undefined) of + ok -> + {reply, ok, #{ + coordinator_pid => CoordinatorPid, + eviction_agent_pid => EvictionAgentPid + }}; + {error, eviction_agent_busy} -> + true = unlink(EvictionAgentPid), + true = unlink(CoordinatorPid), + {reply, {error, eviction_agent_busy}, St} + end + end; +handle_call({disable, CoordinatorPid}, _From, St) -> + case St of + #{ + coordinator_pid := CoordinatorPid, + eviction_agent_pid := EvictionAgentPid + } -> + _ = emqx_eviction_agent:disable(?ENABLE_KIND), + true = unlink(EvictionAgentPid), + true = unlink(CoordinatorPid), + NewSt = maps:without( + [coordinator_pid, eviction_agent_pid], + St + ), + {reply, ok, NewSt}; + #{coordinator_pid := _CoordinatorPid} -> + {reply, {error, invalid_coordinator}, St}; + #{} -> + {reply, {error, already_disabled}, St} + end; +handle_call(status, _From, St) -> + case St of + #{coordinator_pid := Pid} -> + {reply, {enabled, Pid}, St}; + _ -> + {reply, disabled, St} + end; +handle_call(Msg, _From, St) -> + ?SLOG(warning, #{ + msg => "unknown_call", + call => Msg, + state => St + }), + {reply, ignored, St}. + +handle_info(Msg, St) -> + ?SLOG(warning, #{ + msg => "unknown_info", + info => Msg, + state => St + }), + {noreply, St}. + +handle_cast(Msg, St) -> + ?SLOG(warning, #{ + msg => "unknown_cast", + cast => Msg, + state => St + }), + {noreply, St}. + +code_change(_Vsn, State, _Extra) -> + {ok, State}. diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_api.erl b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_api.erl new file mode 100644 index 000000000..fa322d146 --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_api.erl @@ -0,0 +1,738 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_node_rebalance_api). + +-behaviour(minirest_api). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% Swagger specs from hocon schema +-export([ + api_spec/0, + paths/0, + schema/1, + namespace/0 +]). + +-export([ + fields/1, + roots/0 +]). + +%% API callbacks +-export([ + '/load_rebalance/status'/2, + '/load_rebalance/global_status'/2, + '/load_rebalance/availability_check'/2, + '/load_rebalance/:node/start'/2, + '/load_rebalance/:node/stop'/2, + '/load_rebalance/:node/evacuation/start'/2, + '/load_rebalance/:node/evacuation/stop'/2 +]). + +%% Schema examples +-export([ + rebalance_example/0, + rebalance_evacuation_example/0, + translate/2 +]). + +-import(hoconsc, [mk/2, ref/1, ref/2]). +-import(emqx_dashboard_swagger, [error_codes/2]). + +-define(BAD_REQUEST, 'BAD_REQUEST'). +-define(NODE_UNAVAILABLE, 'NODE_UNAVAILABLE'). +-define(NODE_EVACUATING, 'NODE_EVACUATING'). +-define(RPC_ERROR, 'RPC_ERROR'). + +%%-------------------------------------------------------------------- +%% API Spec +%%-------------------------------------------------------------------- + +namespace() -> "load_rebalance". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + [ + "/load_rebalance/status", + "/load_rebalance/global_status", + "/load_rebalance/availability_check", + "/load_rebalance/:node/start", + "/load_rebalance/:node/stop", + "/load_rebalance/:node/evacuation/start", + "/load_rebalance/:node/evacuation/stop" + ]. + +schema("/load_rebalance/status") -> + #{ + 'operationId' => '/load_rebalance/status', + get => #{ + tags => [<<"load_rebalance">>], + summary => <<"Get rebalance status">>, + description => ?DESC("load_rebalance_status"), + responses => #{ + 200 => local_status_response_schema() + } + } + }; +schema("/load_rebalance/global_status") -> + #{ + 'operationId' => '/load_rebalance/global_status', + get => #{ + tags => [<<"load_rebalance">>], + summary => <<"Get global rebalance status">>, + description => ?DESC("load_rebalance_global_status"), + responses => #{ + 200 => response_schema() + } + } + }; +schema("/load_rebalance/availability_check") -> + #{ + 'operationId' => '/load_rebalance/availability_check', + get => #{ + tags => [<<"load_rebalance">>], + summary => <<"Node rebalance availability check">>, + description => ?DESC("load_rebalance_availability_check"), + responses => #{ + 200 => response_schema(), + 503 => error_codes([?NODE_EVACUATING], <<"Node Evacuating">>) + } + } + }; +schema("/load_rebalance/:node/start") -> + #{ + 'operationId' => '/load_rebalance/:node/start', + post => #{ + tags => [<<"load_rebalance">>], + summary => <<"Start rebalancing with the node as coordinator">>, + description => ?DESC("load_rebalance_start"), + parameters => [param_node()], + 'requestBody' => + emqx_dashboard_swagger:schema_with_examples( + ref(rebalance_start), + rebalance_example() + ), + responses => #{ + 200 => response_schema(), + 400 => error_codes([?BAD_REQUEST, ?NODE_UNAVAILABLE], <<"Bad Request">>) + } + } + }; +schema("/load_rebalance/:node/stop") -> + #{ + 'operationId' => '/load_rebalance/:node/stop', + post => #{ + tags => [<<"load_rebalance">>], + summary => <<"Stop rebalancing coordinated by the node">>, + description => ?DESC("load_rebalance_stop"), + parameters => [param_node()], + responses => #{ + 200 => response_schema(), + 400 => error_codes([?BAD_REQUEST, ?NODE_UNAVAILABLE], <<"Bad Request">>) + } + } + }; +schema("/load_rebalance/:node/evacuation/start") -> + #{ + 'operationId' => '/load_rebalance/:node/evacuation/start', + post => #{ + tags => [<<"load_rebalance">>], + summary => <<"Start evacuation on a node">>, + description => ?DESC("load_rebalance_evacuation_start"), + parameters => [param_node()], + 'requestBody' => + emqx_dashboard_swagger:schema_with_examples( + ref(rebalance_evacuation_start), + rebalance_evacuation_example() + ), + responses => #{ + 200 => response_schema(), + 400 => error_codes([?BAD_REQUEST, ?NODE_UNAVAILABLE], <<"Bad Request">>) + } + } + }; +schema("/load_rebalance/:node/evacuation/stop") -> + #{ + 'operationId' => '/load_rebalance/:node/evacuation/stop', + post => #{ + tags => [<<"load_rebalance">>], + summary => <<"Stop evacuation on a node">>, + description => ?DESC("load_rebalance_evacuation_stop"), + parameters => [param_node()], + responses => #{ + 200 => response_schema(), + 400 => error_codes([?BAD_REQUEST, ?NODE_UNAVAILABLE], <<"Bad Request">>) + } + } + }. + +%%-------------------------------------------------------------------- +%% Handlers +%%-------------------------------------------------------------------- + +'/load_rebalance/status'(get, #{}) -> + case emqx_node_rebalance_status:local_status() of + disabled -> + {200, #{status => disabled}}; + {rebalance, Stats} -> + {200, format_status(rebalance, Stats)}; + {evacuation, Stats} -> + {200, format_status(evacuation, Stats)} + end. + +'/load_rebalance/global_status'(get, #{}) -> + #{ + evacuations := Evacuations, + rebalances := Rebalances + } = emqx_node_rebalance_status:global_status(), + {200, #{ + evacuations => format_as_map_list(Evacuations), + rebalances => format_as_map_list(Rebalances) + }}. + +'/load_rebalance/availability_check'(get, #{}) -> + case emqx_eviction_agent:status() of + disabled -> + {200, #{}}; + {enabled, _Stats} -> + error_response(503, ?NODE_EVACUATING, <<"Node Evacuating">>) + end. + +'/load_rebalance/:node/start'(post, #{bindings := #{node := NodeBin}, body := Params0}) -> + with_node(NodeBin, fun(Node) -> + Params1 = translate(rebalance_start, Params0), + with_nodes_at_key(nodes, Params1, fun(Params2) -> + wrap_rpc( + Node, emqx_node_rebalance_api_proto_v1:node_rebalance_start(Node, Params2) + ) + end) + end). + +'/load_rebalance/:node/stop'(post, #{bindings := #{node := NodeBin}}) -> + with_node(NodeBin, fun(Node) -> + wrap_rpc( + Node, emqx_node_rebalance_api_proto_v1:node_rebalance_stop(Node) + ) + end). + +'/load_rebalance/:node/evacuation/start'(post, #{ + bindings := #{node := NodeBin}, body := Params0 +}) -> + with_node(NodeBin, fun(Node) -> + Params1 = translate(rebalance_evacuation_start, Params0), + with_nodes_at_key(migrate_to, Params1, fun(Params2) -> + wrap_rpc( + Node, + emqx_node_rebalance_api_proto_v1:node_rebalance_evacuation_start( + Node, Params2 + ) + ) + end) + end). + +'/load_rebalance/:node/evacuation/stop'(post, #{bindings := #{node := NodeBin}}) -> + with_node(NodeBin, fun(Node) -> + wrap_rpc( + Node, emqx_node_rebalance_api_proto_v1:node_rebalance_evacuation_stop(Node) + ) + end). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +wrap_rpc(Node, RPCResult) -> + case RPCResult of + ok -> + {200, #{}}; + {error, Reason} -> + error_response( + 400, ?BAD_REQUEST, io_lib:format("error on node ~p: ~p", [Node, Reason]) + ); + {badrpc, Reason} -> + error_response( + 503, ?RPC_ERROR, io_lib:format("RPC error on node ~p: ~p", [Node, Reason]) + ) + end. + +format_status(Process, Stats) -> + Stats#{process => Process, status => enabled}. + +validate_nodes(Key, Params) when is_map_key(Key, Params) -> + BinNodes = maps:get(Key, Params), + {ValidNodes, InvalidNodes} = lists:foldl( + fun(BinNode, {Nodes, UnknownNodes}) -> + case parse_node(BinNode) of + {ok, Node} -> {[Node | Nodes], UnknownNodes}; + {error, _} -> {Nodes, [BinNode | UnknownNodes]} + end + end, + {[], []}, + BinNodes + ), + case InvalidNodes of + [] -> + case emqx_node_rebalance_evacuation:available_nodes(ValidNodes) of + ValidNodes -> {ok, Params#{Key => ValidNodes}}; + OtherNodes -> {error, {unavailable, ValidNodes -- OtherNodes}} + end; + _ -> + {error, {invalid, InvalidNodes}} + end; +validate_nodes(_Key, Params) -> + {ok, Params}. + +with_node(BinNode, Fun) -> + case parse_node(BinNode) of + {ok, Node} -> Fun(Node); + {error, _} -> error_response(400, ?BAD_REQUEST, [<<"Invalid node: ">>, BinNode]) + end. + +with_nodes_at_key(Key, Params, Fun) -> + Res = validate_nodes(Key, Params), + case Res of + {ok, Params1} -> + Fun(Params1); + {error, {unavailable, Nodes}} -> + error_response(400, ?NODE_UNAVAILABLE, io_lib:format("Nodes unavailable: ~p", [Nodes])); + {error, {invalid, Nodes}} -> + error_response(400, ?BAD_REQUEST, io_lib:format("Invalid nodes: ~p", [Nodes])) + end. + +parse_node(Bin) when is_binary(Bin) -> + try + {ok, binary_to_existing_atom(Bin)} + catch + error:badarg -> + {error, {unknown, Bin}} + end. + +format_as_map_list(List) -> + lists:map( + fun({Node, Info}) -> + Info#{node => Node} + end, + List + ). + +error_response(HttpCode, Code, Message) -> + {HttpCode, #{ + code => atom_to_binary(Code), + message => iolist_to_binary(Message) + }}. + +without(Keys, Props) -> + lists:filter( + fun({Key, _}) -> + not lists:member(Key, Keys) + end, + Props + ). + +%%------------------------------------------------------------------------------ +%% Schema +%%------------------------------------------------------------------------------ + +translate(Ref, Conf) -> + Options = #{atom_key => true}, + #{Ref := TranslatedConf} = hocon_tconf:check_plain( + ?MODULE, #{atom_to_binary(Ref) => Conf}, Options, [Ref] + ), + TranslatedConf. + +param_node() -> + { + node, + mk(binary(), #{ + in => path, + desc => ?DESC(param_node), + required => true + }) + }. + +fields(rebalance_start) -> + [ + {"wait_health_check", + mk( + emqx_schema:duration_s(), + #{ + desc => ?DESC(wait_health_check), + required => false + } + )}, + {"conn_evict_rate", + mk( + pos_integer(), + #{ + desc => ?DESC(conn_evict_rate), + required => false + } + )}, + {"sess_evict_rate", + mk( + pos_integer(), + #{ + desc => ?DESC(sess_evict_rate), + required => false + } + )}, + {"abs_conn_threshold", + mk( + pos_integer(), + #{ + desc => ?DESC(abs_conn_threshold), + required => false + } + )}, + {"rel_conn_threshold", + mk( + number(), + #{ + desc => ?DESC(rel_conn_threshold), + required => false, + validator => [fun(Value) -> Value > 1.0 end] + } + )}, + {"abs_sess_threshold", + mk( + pos_integer(), + #{ + desc => ?DESC(abs_sess_threshold), + required => false + } + )}, + {"rel_sess_threshold", + mk( + number(), + #{ + desc => ?DESC(rel_sess_threshold), + required => false, + validator => [fun(Value) -> Value > 1.0 end] + } + )}, + {"wait_takeover", + mk( + emqx_schema:duration_s(), + #{ + desc => ?DESC(wait_takeover), + required => false + } + )}, + {"nodes", + mk( + list(binary()), + #{ + desc => ?DESC(rebalance_nodes), + required => false, + validator => [fun(Values) -> length(Values) > 0 end] + } + )} + ]; +fields(rebalance_evacuation_start) -> + [ + {"conn_evict_rate", + mk( + pos_integer(), + #{ + desc => ?DESC(conn_evict_rate), + required => false + } + )}, + {"sess_evict_rate", + mk( + pos_integer(), + #{ + desc => ?DESC(sess_evict_rate), + required => false + } + )}, + {"redirect_to", + mk( + binary(), + #{ + desc => ?DESC(redirect_to), + required => false + } + )}, + {"wait_takeover", + mk( + pos_integer(), + #{ + desc => ?DESC(wait_takeover), + required => false + } + )}, + {"migrate_to", + mk( + list(binary()), + #{ + desc => ?DESC(migrate_to), + required => false, + validator => [fun(Values) -> length(Values) > 0 end] + } + )} + ]; +fields(local_status_disabled) -> + [ + {"status", + mk( + disabled, + #{ + desc => ?DESC(local_status_enabled), + required => true + } + )} + ]; +fields(local_status_enabled) -> + [ + {"status", + mk( + enabled, + #{ + desc => ?DESC(local_status_enabled), + required => true + } + )}, + {"process", + mk( + hoconsc:union([rebalance, evacuation]), + #{ + desc => ?DESC(local_status_process), + required => true + } + )}, + {"state", + mk( + atom(), + #{ + desc => ?DESC(local_status_state), + required => true + } + )}, + {"coordinator_node", + mk( + binary(), + #{ + desc => ?DESC(local_status_coordinator_node), + required => false + } + )}, + {"connection_eviction_rate", + mk( + pos_integer(), + #{ + desc => ?DESC(local_status_connection_eviction_rate), + required => false + } + )}, + {"session_eviction_rate", + mk( + pos_integer(), + #{ + desc => ?DESC(local_status_session_eviction_rate), + required => false + } + )}, + {"connection_goal", + mk( + non_neg_integer(), + #{ + desc => ?DESC(local_status_connection_goal), + required => false + } + )}, + {"session_goal", + mk( + non_neg_integer(), + #{ + desc => ?DESC(local_status_session_goal), + required => false + } + )}, + {"disconnected_session_goal", + mk( + non_neg_integer(), + #{ + desc => ?DESC(local_status_disconnected_session_goal), + required => false + } + )}, + {"session_recipients", + mk( + list(binary()), + #{ + desc => ?DESC(local_status_session_recipients), + required => false + } + )}, + {"recipients", + mk( + list(binary()), + #{ + desc => ?DESC(local_status_recipients), + required => false + } + )}, + {"stats", + mk( + ref(status_stats), + #{ + desc => ?DESC(local_status_stats), + required => false + } + )} + ]; +fields(status_stats) -> + [ + {"initial_connected", + mk( + non_neg_integer(), + #{ + desc => ?DESC(status_stats_initial_connected), + required => true + } + )}, + {"current_connected", + mk( + non_neg_integer(), + #{ + desc => ?DESC(status_stats_current_connected), + required => true + } + )}, + {"initial_sessions", + mk( + non_neg_integer(), + #{ + desc => ?DESC(status_stats_initial_sessions), + required => true + } + )}, + {"current_sessions", + mk( + non_neg_integer(), + #{ + desc => ?DESC(status_stats_current_sessions), + required => true + } + )}, + {"current_disconnected_sessions", + mk( + non_neg_integer(), + #{ + desc => ?DESC(status_stats_current_disconnected_sessions), + required => false + } + )} + ]; +fields(global_coordinator_status) -> + without( + ["status", "process", "session_goal", "session_recipients", "stats"], + fields(local_status_enabled) + ) ++ + [ + {"donors", + mk( + list(binary()), + #{ + desc => ?DESC(coordinator_status_donors), + required => false + } + )}, + {"donor_conn_avg", + mk( + non_neg_integer(), + #{ + desc => ?DESC(coordinator_status_donor_conn_avg), + required => false + } + )}, + {"donor_sess_avg", + mk( + non_neg_integer(), + #{ + desc => ?DESC(coordinator_status_donor_sess_avg), + required => false + } + )}, + {"node", + mk( + binary(), + #{ + desc => ?DESC(coordinator_status_node), + required => true + } + )} + ]; +fields(global_evacuation_status) -> + without(["status", "process"], fields(local_status_enabled)) ++ + [ + {"node", + mk( + binary(), + #{ + desc => ?DESC(evacuation_status_node), + required => true + } + )} + ]; +fields(global_status) -> + [ + {"evacuations", + mk( + hoconsc:array(ref(global_evacuation_status)), + #{ + desc => ?DESC(global_status_evacuations), + required => false + } + )}, + {"rebalances", + mk( + hoconsc:array(ref(global_coordinator_status)), + #{ + desc => ?DESC(global_status_rebalances), + required => false + } + )} + ]. + +rebalance_example() -> + #{ + wait_health_check => 10, + conn_evict_rate => 10, + sess_evict_rate => 20, + abs_conn_threshold => 10, + rel_conn_threshold => 1.5, + abs_sess_threshold => 10, + rel_sess_threshold => 1.5, + wait_takeover => 10, + nodes => [<<"othernode@127.0.0.1">>] + }. + +rebalance_evacuation_example() -> + #{ + conn_evict_rate => 100, + sess_evict_rate => 100, + redirect_to => <<"othernode:1883">>, + wait_takeover => 10, + migrate_to => [<<"othernode@127.0.0.1">>] + }. + +local_status_response_schema() -> + hoconsc:union([ref(local_status_disabled), ref(local_status_enabled)]). + +response_schema() -> + mk( + map(), + #{ + desc => ?DESC(empty_response) + } + ). + +roots() -> []. diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_app.erl b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_app.erl new file mode 100644 index 000000000..3cd59e0f4 --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_app.erl @@ -0,0 +1,22 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_app). + +-behaviour(application). + +-emqx_plugin(?MODULE). + +-export([ + start/2, + stop/1 +]). + +start(_Type, _Args) -> + {ok, Sup} = emqx_node_rebalance_sup:start_link(), + ok = emqx_node_rebalance_cli:load(), + {ok, Sup}. + +stop(_State) -> + emqx_node_rebalance_cli:unload(). diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl new file mode 100644 index 000000000..a2706f13b --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl @@ -0,0 +1,305 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_cli). + +%% APIs +-export([ + load/0, + unload/0, + cli/1 +]). + +load() -> + emqx_ctl:register_command(rebalance, {?MODULE, cli}, []). + +unload() -> + emqx_ctl:unregister_command(rebalance). + +cli(["start" | StartArgs]) -> + case start_args(StartArgs) of + {evacuation, Opts} -> + case emqx_node_rebalance_evacuation:status() of + disabled -> + ok = emqx_node_rebalance_evacuation:start(Opts), + emqx_ctl:print("Rebalance(evacuation) started~n"), + true; + {enabled, _} -> + emqx_ctl:print("Rebalance is already enabled~n"), + false + end; + {rebalance, Opts} -> + case emqx_node_rebalance:start(Opts) of + ok -> + emqx_ctl:print("Rebalance started~n"), + true; + {error, Reason} -> + emqx_ctl:print("Rebalance start error: ~p~n", [Reason]), + false + end; + {error, Error} -> + emqx_ctl:print("Rebalance start error: ~s~n", [Error]), + false + end; +cli(["node-status", NodeStr]) -> + case emqx_misc:safe_to_existing_atom(NodeStr, utf8) of + {ok, Node} -> + node_status(emqx_node_rebalance_status:local_status(Node)); + {error, _} -> + emqx_ctl:print("Node status error: invalid node~n"), + false + end; +cli(["node-status"]) -> + node_status(emqx_node_rebalance_status:local_status()); +cli(["status"]) -> + #{ + evacuations := Evacuations, + rebalances := Rebalances + } = emqx_node_rebalance_status:global_status(), + lists:foreach( + fun({Node, Status}) -> + emqx_ctl:print( + "--------------------------------------------------------------------~n" + ), + emqx_ctl:print( + "Node ~p: evacuation~n~s", + [Node, emqx_node_rebalance_status:format_local_status(Status)] + ) + end, + Evacuations + ), + lists:foreach( + fun({Node, Status}) -> + emqx_ctl:print( + "--------------------------------------------------------------------~n" + ), + emqx_ctl:print( + "Node ~p: rebalance coordinator~n~s", + [Node, emqx_node_rebalance_status:format_coordinator_status(Status)] + ) + end, + Rebalances + ); +cli(["stop"]) -> + case emqx_node_rebalance_evacuation:status() of + {enabled, _} -> + ok = emqx_node_rebalance_evacuation:stop(), + emqx_ctl:print("Rebalance(evacuation) stopped~n"), + true; + disabled -> + case emqx_node_rebalance:status() of + {enabled, _} -> + ok = emqx_node_rebalance:stop(), + emqx_ctl:print("Rebalance stopped~n"), + true; + disabled -> + emqx_ctl:print("Rebalance is already disabled~n"), + false + end + end; +cli(_) -> + emqx_ctl:usage( + [ + { + "rebalance start --evacuation \\\n" + " [--redirect-to \"Host1:Port1 Host2:Port2 ...\"] \\\n" + " [--conn-evict-rate CountPerSec] \\\n" + " [--migrate-to \"node1@host1 node2@host2 ...\"] \\\n" + " [--wait-takeover Secs] \\\n" + " [--sess-evict-rate CountPerSec]", + "Start current node evacuation with optional server redirect to the specified servers" + }, + + { + "rebalance start \\\n" + " [--nodes \"node1@host1 node2@host2\"] \\\n" + " [--wait-health-check Secs] \\\n" + " [--conn-evict-rate ConnPerSec] \\\n" + " [--abs-conn-threshold Count] \\\n" + " [--rel-conn-threshold Fraction] \\\n" + " [--conn-evict-rate ConnPerSec] \\\n" + " [--wait-takeover Secs] \\\n" + " [--sess-evict-rate CountPerSec] \\\n" + " [--abs-sess-threshold Count] \\\n" + " [--rel-sess-threshold Fraction]", + "Start rebalance on the specified nodes using the current node as the coordinator" + }, + + {"rebalance node-status", "Get current node rebalance status"}, + + {"rebalance node-status \"node1@host1\"", "Get remote node rebalance status"}, + + {"rebalance status", + "Get statuses of all current rebalance/evacuation processes across the cluster"}, + + {"rebalance stop", "Stop node rebalance"} + ] + ). + +node_status(NodeStatus) -> + case NodeStatus of + {Process, Status} when Process =:= evacuation orelse Process =:= rebalance -> + emqx_ctl:print( + "Rebalance type: ~p~n~s~n", + [Process, emqx_node_rebalance_status:format_local_status(Status)] + ); + disabled -> + emqx_ctl:print("Rebalance disabled~n"); + Other -> + emqx_ctl:print("Error detecting rebalance status: ~p~n", [Other]) + end. + +start_args(Args) -> + case collect_args(Args, #{}) of + {ok, #{"--evacuation" := true} = Collected} -> + case validate_evacuation(maps:to_list(Collected), #{}) of + {ok, Validated} -> + {evacuation, Validated}; + {error, _} = Error -> + Error + end; + {ok, #{} = Collected} -> + case validate_rebalance(maps:to_list(Collected), #{}) of + {ok, Validated} -> + {rebalance, Validated}; + {error, _} = Error -> + Error + end; + {error, _} = Error -> + Error + end. + +collect_args([], Map) -> + {ok, Map}; +%% evacuation +collect_args(["--evacuation" | Args], Map) -> + collect_args(Args, Map#{"--evacuation" => true}); +collect_args(["--redirect-to", ServerReference | Args], Map) -> + collect_args(Args, Map#{"--redirect-to" => ServerReference}); +collect_args(["--migrate-to", MigrateTo | Args], Map) -> + collect_args(Args, Map#{"--migrate-to" => MigrateTo}); +%% rebalance +collect_args(["--nodes", Nodes | Args], Map) -> + collect_args(Args, Map#{"--nodes" => Nodes}); +collect_args(["--wait-health-check", WaitHealthCheck | Args], Map) -> + collect_args(Args, Map#{"--wait-health-check" => WaitHealthCheck}); +collect_args(["--abs-conn-threshold", AbsConnThres | Args], Map) -> + collect_args(Args, Map#{"--abs-conn-threshold" => AbsConnThres}); +collect_args(["--rel-conn-threshold", RelConnThres | Args], Map) -> + collect_args(Args, Map#{"--rel-conn-threshold" => RelConnThres}); +collect_args(["--abs-sess-threshold", AbsSessThres | Args], Map) -> + collect_args(Args, Map#{"--abs-sess-threshold" => AbsSessThres}); +collect_args(["--rel-sess-threshold", RelSessThres | Args], Map) -> + collect_args(Args, Map#{"--rel-sess-threshold" => RelSessThres}); +%% common +collect_args(["--conn-evict-rate", ConnEvictRate | Args], Map) -> + collect_args(Args, Map#{"--conn-evict-rate" => ConnEvictRate}); +collect_args(["--wait-takeover", WaitTakeover | Args], Map) -> + collect_args(Args, Map#{"--wait-takeover" => WaitTakeover}); +collect_args(["--sess-evict-rate", SessEvictRate | Args], Map) -> + collect_args(Args, Map#{"--sess-evict-rate" => SessEvictRate}); +%% fallback +collect_args(Args, _Map) -> + {error, io_lib:format("unknown arguments: ~p", [Args])}. + +validate_evacuation([], Map) -> + {ok, Map}; +validate_evacuation([{"--evacuation", _} | Rest], Map) -> + validate_evacuation(Rest, Map); +validate_evacuation([{"--redirect-to", ServerReference} | Rest], Map) -> + validate_evacuation(Rest, Map#{server_reference => list_to_binary(ServerReference)}); +validate_evacuation([{"--conn-evict-rate", _} | _] = Opts, Map) -> + validate_pos_int(conn_evict_rate, Opts, Map, fun validate_evacuation/2); +validate_evacuation([{"--sess-evict-rate", _} | _] = Opts, Map) -> + validate_pos_int(sess_evict_rate, Opts, Map, fun validate_evacuation/2); +validate_evacuation([{"--wait-takeover", _} | _] = Opts, Map) -> + validate_pos_int(wait_takeover, Opts, Map, fun validate_evacuation/2); +validate_evacuation([{"--migrate-to", MigrateTo} | Rest], Map) -> + case strings_to_atoms(string:tokens(MigrateTo, ", ")) of + {_, Invalid} when Invalid =/= [] -> + {error, io_lib:format("invalid --migrate-to, invalid nodes: ~p", [Invalid])}; + {Nodes, []} -> + case emqx_node_rebalance_evacuation:available_nodes(Nodes) of + [] -> + {error, "invalid --migrate-to, no nodes"}; + Nodes -> + validate_evacuation(Rest, Map#{migrate_to => Nodes}); + OtherNodes -> + {error, + io_lib:format( + "invalid --migrate-to, unavailable nodes: ~p", + [Nodes -- OtherNodes] + )} + end + end; +validate_evacuation(Rest, _Map) -> + {error, io_lib:format("unknown evacuation arguments: ~p", [Rest])}. + +validate_rebalance([], Map) -> + {ok, Map}; +validate_rebalance([{"--wait-health-check", _} | _] = Opts, Map) -> + validate_pos_int(wait_health_check, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--conn-evict-rate", _} | _] = Opts, Map) -> + validate_pos_int(conn_evict_rate, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--sess-evict-rate", _} | _] = Opts, Map) -> + validate_pos_int(sess_evict_rate, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--abs-conn-threshold", _} | _] = Opts, Map) -> + validate_pos_int(abs_conn_threshold, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--rel-conn-threshold", _} | _] = Opts, Map) -> + validate_fraction(rel_conn_threshold, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--abs-sess-threshold", _} | _] = Opts, Map) -> + validate_pos_int(abs_sess_threshold, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--rel-sess-threshold", _} | _] = Opts, Map) -> + validate_fraction(rel_sess_threshold, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--wait-takeover", _} | _] = Opts, Map) -> + validate_pos_int(wait_takeover, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--nodes", NodeStr} | Rest], Map) -> + case strings_to_atoms(string:tokens(NodeStr, ", ")) of + {_, Invalid} when Invalid =/= [] -> + {error, io_lib:format("invalid --nodes, invalid nodes: ~p", [Invalid])}; + {Nodes, []} -> + case emqx_node_rebalance:available_nodes(Nodes) of + [] -> + {error, "invalid --nodes, no nodes"}; + Nodes -> + validate_rebalance(Rest, Map#{nodes => Nodes}); + OtherNodes -> + {error, + io_lib:format( + "invalid --nodes, unavailable nodes: ~p", + [Nodes -- OtherNodes] + )} + end + end; +validate_rebalance(Rest, _Map) -> + {error, io_lib:format("unknown rebalance arguments: ~p", [Rest])}. + +validate_fraction(Name, [{OptionName, Value} | Rest], Map, Next) -> + case string:to_float(Value) of + {Num, ""} when Num > 1.0 -> + Next(Rest, Map#{Name => Num}); + _ -> + {error, "invalid " ++ OptionName ++ " value"} + end. + +validate_pos_int(Name, [{OptionName, Value} | Rest], Map, Next) -> + case string:to_integer(Value) of + {Int, ""} when Int > 0 -> + Next(Rest, Map#{Name => Int}); + _ -> + {error, "invalid " ++ OptionName ++ " value"} + end. + +strings_to_atoms(Strings) -> + strings_to_atoms(Strings, [], []). + +strings_to_atoms([], Atoms, Invalid) -> + {lists:reverse(Atoms), lists:reverse(Invalid)}; +strings_to_atoms([Str | Rest], Atoms, Invalid) -> + case emqx_misc:safe_to_existing_atom(Str, utf8) of + {ok, Atom} -> + strings_to_atoms(Rest, [Atom | Atoms], Invalid); + {error, _} -> + strings_to_atoms(Rest, Atoms, [Str | Invalid]) + end. diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl new file mode 100644 index 000000000..4de362ca9 --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl @@ -0,0 +1,308 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_evacuation). + +-include("emqx_node_rebalance.hrl"). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-export([ + start/1, + status/0, + stop/0 +]). + +-export([start_link/0]). + +-behaviour(gen_statem). + +-export([ + init/1, + callback_mode/0, + handle_event/4, + code_change/4 +]). + +-export([ + is_node_available/0, + available_nodes/1 +]). + +-export_type([ + start_opts/0, + start_error/0 +]). + +-ifdef(TEST). +-export([migrate_to/1]). +-endif. + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +-define(EVICT_INTERVAL_NO_NODES, 30000). + +-type migrate_to() :: [node()] | undefined. + +-type start_opts() :: #{ + server_reference => emqx_eviction_agent:server_reference(), + conn_evict_rate => pos_integer(), + sess_evict_rate => pos_integer(), + wait_takeover => pos_integer(), + migrate_to => migrate_to() +}. +-type start_error() :: already_started | eviction_agent_busy. +-type stats() :: #{ + initial_conns := non_neg_integer(), + initial_sessions := non_neg_integer(), + current_conns := non_neg_integer(), + current_sessions := non_neg_integer(), + conn_evict_rate := pos_integer(), + sess_evict_rate := pos_integer(), + server_reference := emqx_eviction_agent:server_reference(), + migrate_to := migrate_to() +}. +-type status() :: {enabled, stats()} | disabled. + +-spec start(start_opts()) -> ok_or_error(start_error()). +start(StartOpts) -> + Opts = maps:merge(default_opts(), StartOpts), + gen_statem:call(?MODULE, {start, Opts}). + +-spec stop() -> ok_or_error(not_started). +stop() -> + gen_statem:call(?MODULE, stop). + +-spec status() -> status(). +status() -> + gen_statem:call(?MODULE, status). + +-spec start_link() -> startlink_ret(). +start_link() -> + gen_statem:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec available_nodes(list(node())) -> list(node()). +available_nodes(Nodes) when is_list(Nodes) -> + {Available, _} = emqx_node_rebalance_evacuation_proto_v1:available_nodes(Nodes), + lists:filter(fun is_atom/1, Available). + +%%-------------------------------------------------------------------- +%% gen_statem callbacks +%%-------------------------------------------------------------------- + +callback_mode() -> handle_event_function. + +%% states: disabled, evicting_conns, waiting_takeover, evicting_sessions, prohibiting + +init([]) -> + case emqx_node_rebalance_evacuation_persist:read(default_opts()) of + {ok, #{server_reference := ServerReference} = Opts} -> + ?SLOG(warning, #{msg => "restoring_evacuation_state", opts => Opts}), + case emqx_eviction_agent:enable(?MODULE, ServerReference) of + ok -> + Data = init_data(#{}, Opts), + ok = warn_enabled(), + {ok, evicting_conns, Data, [{state_timeout, 0, evict_conns}]}; + {error, eviction_agent_busy} -> + emqx_node_rebalance_evacuation_persist:clear(), + {ok, disabled, #{}} + end; + none -> + {ok, disabled, #{}} + end. + +%% start +handle_event( + {call, From}, + {start, #{server_reference := ServerReference} = Opts}, + disabled, + #{} = Data +) -> + case emqx_eviction_agent:enable(?MODULE, ServerReference) of + ok -> + NewData = init_data(Data, Opts), + ok = emqx_node_rebalance_evacuation_persist:save(Opts), + ?SLOG(warning, #{ + msg => "node_evacuation_started", + opts => Opts + }), + {next_state, evicting_conns, NewData, [ + {state_timeout, 0, evict_conns}, + {reply, From, ok} + ]}; + {error, eviction_agent_busy} -> + {keep_state_and_data, [{reply, From, {error, eviction_agent_busy}}]} + end; +handle_event({call, From}, {start, _Opts}, _State, #{}) -> + {keep_state_and_data, [{reply, From, {error, already_started}}]}; +%% stop +handle_event({call, From}, stop, disabled, #{}) -> + {keep_state_and_data, [{reply, From, {error, not_started}}]}; +handle_event({call, From}, stop, _State, Data) -> + ok = emqx_node_rebalance_evacuation_persist:clear(), + _ = emqx_eviction_agent:disable(?MODULE), + ?SLOG(warning, #{msg => "node_evacuation_stopped"}), + {next_state, disabled, deinit(Data), [{reply, From, ok}]}; +%% status +handle_event({call, From}, status, disabled, #{}) -> + {keep_state_and_data, [{reply, From, disabled}]}; +handle_event({call, From}, status, State, #{migrate_to := MigrateTo} = Data) -> + Stats = maps:with( + [ + initial_conns, + current_conns, + initial_sessions, + current_sessions, + server_reference, + conn_evict_rate, + sess_evict_rate + ], + Data + ), + {keep_state_and_data, [ + {reply, From, {enabled, Stats#{state => State, migrate_to => migrate_to(MigrateTo)}}} + ]}; +%% conn eviction +handle_event( + state_timeout, + evict_conns, + evicting_conns, + #{ + conn_evict_rate := ConnEvictRate, + wait_takeover := WaitTakeover + } = Data +) -> + case emqx_eviction_agent:status() of + {enabled, #{connections := Conns}} when Conns > 0 -> + ok = emqx_eviction_agent:evict_connections(ConnEvictRate), + ?tp(debug, node_evacuation_evict_conn, #{conn_evict_rate => ConnEvictRate}), + ?SLOG( + warning, + #{ + msg => "node_evacuation_evict_conns", + count => Conns, + conn_evict_rate => ConnEvictRate + } + ), + NewData = Data#{current_conns => Conns}, + {keep_state, NewData, [{state_timeout, ?EVICT_INTERVAL, evict_conns}]}; + {enabled, #{connections := 0}} -> + NewData = Data#{current_conns => 0}, + ?SLOG(warning, #{msg => "node_evacuation_evict_conns_done"}), + {next_state, waiting_takeover, NewData, [ + {state_timeout, timer:seconds(WaitTakeover), evict_sessions} + ]} + end; +handle_event( + state_timeout, + evict_sessions, + waiting_takeover, + Data +) -> + ?SLOG(warning, #{msg => "node_evacuation_waiting_takeover_done"}), + {next_state, evicting_sessions, Data, [{state_timeout, 0, evict_sessions}]}; +%% session eviction +handle_event( + state_timeout, + evict_sessions, + evicting_sessions, + #{ + sess_evict_rate := SessEvictRate, + migrate_to := MigrateTo, + current_sessions := CurrSessCount + } = Data +) -> + case emqx_eviction_agent:status() of + {enabled, #{sessions := SessCount}} when SessCount > 0 -> + case migrate_to(MigrateTo) of + [] -> + ?SLOG(warning, #{ + msg => "no_nodes_to_evacuate_sessions", session_count => CurrSessCount + }), + {keep_state_and_data, [ + {state_timeout, ?EVICT_INTERVAL_NO_NODES, evict_sessions} + ]}; + Nodes -> + ok = emqx_eviction_agent:evict_sessions(SessEvictRate, Nodes), + ?SLOG( + warning, + #{ + msg => "node_evacuation_evict_sessions", + session_count => SessCount, + session_evict_rate => SessEvictRate, + target_nodes => Nodes + } + ), + NewData = Data#{current_sessions => SessCount}, + {keep_state, NewData, [{state_timeout, ?EVICT_INTERVAL, evict_sessions}]} + end; + {enabled, #{sessions := 0}} -> + ?tp(debug, node_evacuation_evict_sess_over, #{}), + ?SLOG(warning, #{msg => "node_evacuation_evict_sessions_over"}), + NewData = Data#{current_sessions => 0}, + {next_state, prohibiting, NewData} + end; +handle_event({call, From}, Msg, State, Data) -> + ?SLOG(warning, #{msg => "unknown_call", call => Msg, state => State, data => Data}), + {keep_state_and_data, [{reply, From, ignored}]}; +handle_event(info, Msg, State, Data) -> + ?SLOG(warning, #{msg => "unknown_info", info => Msg, state => State, data => Data}), + keep_state_and_data; +handle_event(cast, Msg, State, Data) -> + ?SLOG(warning, #{msg => "unknown_cast", cast => Msg, state => State, data => Data}), + keep_state_and_data. + +code_change(_Vsn, State, Data, _Extra) -> + {ok, State, Data}. + +%%-------------------------------------------------------------------- +%% internal funs +%%-------------------------------------------------------------------- + +default_opts() -> + #{ + server_reference => undefined, + conn_evict_rate => ?DEFAULT_CONN_EVICT_RATE, + sess_evict_rate => ?DEFAULT_SESS_EVICT_RATE, + wait_takeover => ?DEFAULT_WAIT_TAKEOVER, + migrate_to => undefined + }. + +init_data(Data0, Opts) -> + Data1 = maps:merge(Data0, Opts), + {enabled, #{connections := ConnCount, sessions := SessCount}} = emqx_eviction_agent:status(), + Data1#{ + initial_conns => ConnCount, + current_conns => ConnCount, + initial_sessions => SessCount, + current_sessions => SessCount + }. + +deinit(Data) -> + Keys = + [initial_conns, current_conns, initial_sessions, current_sessions] ++ + maps:keys(default_opts()), + maps:without(Keys, Data). + +warn_enabled() -> + ?SLOG(warning, #{msg => "node_evacuation_enabled"}), + io:format( + standard_error, "Node evacuation is enabled. The node will not receive connections.~n", [] + ). + +migrate_to(undefined) -> + migrate_to(all_nodes()); +migrate_to(Nodes) when is_list(Nodes) -> + available_nodes(Nodes). + +is_node_available() -> + disabled = emqx_eviction_agent:status(), + node(). + +all_nodes() -> + mria_mnesia:running_nodes() -- [node()]. diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl new file mode 100644 index 000000000..3fc9faeea --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl @@ -0,0 +1,120 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_evacuation_persist). + +-export([ + save/1, + clear/0, + read/1 +]). + +-ifdef(TEST). +-export([evacuation_filepath/0]). +-endif. + +-include("emqx_node_rebalance.hrl"). +-include_lib("emqx/include/types.hrl"). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +%% do not persist `migrate_to`: +%% * after restart there is nothing to migrate +%% * this value may be invalid after node was offline +-type persisted_start_opts() :: #{ + server_reference => emqx_eviction_agent:server_reference(), + conn_evict_rate => pos_integer(), + sess_evict_rate => pos_integer(), + wait_takeover => pos_integer() +}. +-type start_opts() :: #{ + server_reference => emqx_eviction_agent:server_reference(), + conn_evict_rate => pos_integer(), + sess_evict_rate => pos_integer(), + wait_takeover => pos_integer(), + migrate_to => emqx_node_rebalance_evacuation:migrate_to() +}. + +-spec save(persisted_start_opts()) -> ok_or_error(term()). +save( + #{ + server_reference := ServerReference, + conn_evict_rate := ConnEvictRate, + sess_evict_rate := SessEvictRate, + wait_takeover := WaitTakeover + } = Data +) when + (is_binary(ServerReference) orelse ServerReference =:= undefined) andalso + is_integer(ConnEvictRate) andalso ConnEvictRate > 0 andalso + is_integer(SessEvictRate) andalso SessEvictRate > 0 andalso + is_integer(WaitTakeover) andalso WaitTakeover >= 0 +-> + Filepath = evacuation_filepath(), + case filelib:ensure_dir(Filepath) of + ok -> + JsonData = emqx_json:encode( + prepare_for_encode(maps:with(persist_keys(), Data)), + [pretty] + ), + file:write_file(Filepath, JsonData); + {error, _} = Error -> + Error + end. + +-spec clear() -> ok. +clear() -> + file:delete(evacuation_filepath()). + +-spec read(start_opts()) -> {ok, start_opts()} | none. +read(DefaultOpts) -> + case file:read_file(evacuation_filepath()) of + {ok, Data} -> + case emqx_json:safe_decode(Data, [return_maps]) of + {ok, Map} when is_map(Map) -> + {ok, map_to_opts(DefaultOpts, Map)}; + _NotAMap -> + {ok, DefaultOpts} + end; + {error, _} -> + none + end. + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +persist_keys() -> + [ + server_reference, + conn_evict_rate, + sess_evict_rate, + wait_takeover + ]. + +prepare_for_encode(#{server_reference := undefined} = Data) -> + Data#{server_reference => null}; +prepare_for_encode(Data) -> + Data. + +format_after_decode(#{server_reference := null} = Data) -> + Data#{server_reference => undefined}; +format_after_decode(Data) -> + Data. + +map_to_opts(DefaultOpts, Map) -> + format_after_decode( + map_to_opts( + maps:to_list(DefaultOpts), Map, #{} + ) + ). + +map_to_opts([], _Map, Opts) -> + Opts; +map_to_opts([{Key, DefaultVal} | Rest], Map, Opts) -> + map_to_opts(Rest, Map, Opts#{Key => maps:get(atom_to_binary(Key), Map, DefaultVal)}). + +evacuation_filepath() -> + filename:join([emqx:data_dir(), ?EVACUATION_FILENAME]). diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_status.erl b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_status.erl new file mode 100644 index 000000000..63675a3da --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_status.erl @@ -0,0 +1,238 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_status). + +-export([ + local_status/0, + local_status/1, + global_status/0, + format_local_status/1, + format_coordinator_status/1 +]). + +%% For RPC +-export([ + evacuation_status/0, + rebalance_status/0 +]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +-spec local_status() -> disabled | {evacuation, map()} | {rebalance, map()}. +local_status() -> + case emqx_node_rebalance_evacuation:status() of + {enabled, Status} -> + {evacuation, evacuation(Status)}; + disabled -> + case emqx_node_rebalance_agent:status() of + {enabled, CoordinatorPid} -> + case emqx_node_rebalance:status(CoordinatorPid) of + {enabled, Status} -> + local_rebalance(Status, node()); + disabled -> + disabled + end; + disabled -> + disabled + end + end. + +-spec local_status(node()) -> disabled | {evacuation, map()} | {rebalance, map()}. +local_status(Node) -> + emqx_node_rebalance_status_proto_v1:local_status(Node). + +-spec format_local_status(map()) -> iodata(). +format_local_status(Status) -> + format_status(Status, local_status_field_format_order()). + +-spec global_status() -> #{rebalances := [{node(), map()}], evacuations := [{node(), map()}]}. +global_status() -> + Nodes = mria_mnesia:running_nodes(), + {RebalanceResults, _} = emqx_node_rebalance_status_proto_v1:rebalance_status(Nodes), + Rebalances = [ + {Node, coordinator_rebalance(Status)} + || {Node, {enabled, Status}} <- RebalanceResults + ], + {EvacuatioResults, _} = emqx_node_rebalance_status_proto_v1:evacuation_status(Nodes), + Evacuations = [{Node, evacuation(Status)} || {Node, {enabled, Status}} <- EvacuatioResults], + #{rebalances => Rebalances, evacuations => Evacuations}. + +-spec format_coordinator_status(map()) -> iodata(). +format_coordinator_status(Status) -> + format_status(Status, coordinator_status_field_format_order()). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +evacuation(Status) -> + #{ + state => maps:get(state, Status), + connection_eviction_rate => maps:get(conn_evict_rate, Status), + session_eviction_rate => maps:get(sess_evict_rate, Status), + connection_goal => 0, + session_goal => 0, + session_recipients => maps:get(migrate_to, Status), + stats => #{ + initial_connected => maps:get(initial_conns, Status), + current_connected => maps:get(current_conns, Status), + initial_sessions => maps:get(initial_sessions, Status), + current_sessions => maps:get(current_sessions, Status) + } + }. + +local_rebalance(#{donors := Donors} = Stats, Node) -> + case lists:member(Node, Donors) of + true -> {rebalance, donor_rebalance(Stats, Node)}; + false -> disabled + end. + +donor_rebalance(Status, Node) -> + Opts = maps:get(opts, Status), + InitialConnCounts = maps:get(initial_conn_counts, Status), + InitialSessCounts = maps:get(initial_sess_counts, Status), + + CurrentStats = #{ + initial_connected => maps:get(Node, InitialConnCounts), + initial_sessions => maps:get(Node, InitialSessCounts), + current_connected => emqx_eviction_agent:connection_count(), + current_sessions => emqx_eviction_agent:session_count(), + current_disconnected_sessions => emqx_eviction_agent:session_count( + disconnected + ) + }, + maps:from_list( + [ + {state, maps:get(state, Status)}, + {coordinator_node, maps:get(coordinator_node, Status)}, + {connection_eviction_rate, maps:get(conn_evict_rate, Opts)}, + {session_eviction_rate, maps:get(sess_evict_rate, Opts)}, + {recipients, maps:get(recipients, Status)}, + {stats, CurrentStats} + ] ++ + [ + {connection_goal, maps:get(recipient_conn_avg, Status)} + || maps:is_key(recipient_conn_avg, Status) + ] ++ + [ + {disconnected_session_goal, maps:get(recipient_sess_avg, Status)} + || maps:is_key(recipient_sess_avg, Status) + ] + ). + +coordinator_rebalance(Status) -> + Opts = maps:get(opts, Status), + maps:from_list( + [ + {state, maps:get(state, Status)}, + {coordinator_node, maps:get(coordinator_node, Status)}, + {connection_eviction_rate, maps:get(conn_evict_rate, Opts)}, + {session_eviction_rate, maps:get(sess_evict_rate, Opts)}, + {recipients, maps:get(recipients, Status)}, + {donors, maps:get(donors, Status)} + ] ++ + [ + {connection_goal, maps:get(recipient_conn_avg, Status)} + || maps:is_key(recipient_conn_avg, Status) + ] ++ + [ + {disconnected_session_goal, maps:get(recipient_sess_avg, Status)} + || maps:is_key(recipient_sess_avg, Status) + ] ++ + [ + {donor_conn_avg, maps:get(donor_conn_avg, Status)} + || maps:is_key(donor_conn_avg, Status) + ] ++ + [ + {donor_sess_avg, maps:get(donor_sess_avg, Status)} + || maps:is_key(donor_sess_avg, Status) + ] + ). + +local_status_field_format_order() -> + [ + state, + coordinator_node, + connection_eviction_rate, + session_eviction_rate, + connection_goal, + session_goal, + disconnected_session_goal, + session_recipients, + recipients, + stats + ]. + +coordinator_status_field_format_order() -> + [ + state, + coordinator_node, + donors, + recipients, + connection_eviction_rate, + session_eviction_rate, + connection_goal, + disconnected_session_goal, + donor_conn_avg, + donor_sess_avg + ]. + +format_status(Status, FieldOrder) -> + Fields = lists:flatmap( + fun(FieldName) -> + maps:to_list(maps:with([FieldName], Status)) + end, + FieldOrder + ), + lists:map( + fun format_local_status_field/1, + Fields + ). + +format_local_status_field({state, State}) -> + io_lib:format("Rebalance state: ~p~n", [State]); +format_local_status_field({coordinator_node, Node}) -> + io_lib:format("Coordinator node: ~p~n", [Node]); +format_local_status_field({connection_eviction_rate, ConnEvictRate}) -> + io_lib:format("Connection eviction rate: ~p connections/second~n", [ConnEvictRate]); +format_local_status_field({session_eviction_rate, SessEvictRate}) -> + io_lib:format("Session eviction rate: ~p sessions/second~n", [SessEvictRate]); +format_local_status_field({connection_goal, ConnGoal}) -> + io_lib:format("Connection goal: ~p~n", [ConnGoal]); +format_local_status_field({session_goal, SessGoal}) -> + io_lib:format("Session goal: ~p~n", [SessGoal]); +format_local_status_field({disconnected_session_goal, DisconnSessGoal}) -> + io_lib:format("Disconnected session goal: ~p~n", [DisconnSessGoal]); +format_local_status_field({session_recipients, SessionRecipients}) -> + io_lib:format("Session recipient nodes: ~p~n", [SessionRecipients]); +format_local_status_field({recipients, Recipients}) -> + io_lib:format("Recipient nodes: ~p~n", [Recipients]); +format_local_status_field({donors, Donors}) -> + io_lib:format("Donor nodes: ~p~n", [Donors]); +format_local_status_field({donor_conn_avg, DonorConnAvg}) -> + io_lib:format("Current average donor node connection count: ~p~n", [DonorConnAvg]); +format_local_status_field({donor_sess_avg, DonorSessAvg}) -> + io_lib:format("Current average donor node disconnected session count: ~p~n", [DonorSessAvg]); +format_local_status_field({stats, Stats}) -> + format_local_stats(Stats). + +format_local_stats(Stats) -> + [ + "Channel statistics:\n" + | lists:map( + fun({Name, Value}) -> + io_lib:format(" ~p: ~p~n", [Name, Value]) + end, + maps:to_list(Stats) + ) + ]. + +evacuation_status() -> + {node(), emqx_node_rebalance_evacuation:status()}. + +rebalance_status() -> + {node(), emqx_node_rebalance:status()}. diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl new file mode 100644 index 000000000..cfaccc4c2 --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl @@ -0,0 +1,35 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + Childs = [ + child_spec(emqx_node_rebalance_evacuation, []), + child_spec(emqx_node_rebalance_agent, []), + child_spec(emqx_node_rebalance, []) + ], + {ok, { + #{strategy => one_for_one, intensity => 10, period => 3600}, + Childs + }}. + +child_spec(Mod, Args) -> + #{ + id => Mod, + start => {Mod, start_link, Args}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [Mod] + }. diff --git a/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_api_proto_v1.erl b/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_api_proto_v1.erl new file mode 100644 index 000000000..131973932 --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_api_proto_v1.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_api_proto_v1). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + node_rebalance_evacuation_start/2, + node_rebalance_evacuation_stop/1, + + node_rebalance_start/2, + node_rebalance_stop/1 +]). + +-include_lib("emqx/include/bpapi.hrl"). +-include_lib("emqx/include/types.hrl"). + +introduced_in() -> + "5.0.22". + +-spec node_rebalance_evacuation_start(node(), emqx_node_rebalance_evacuation:start_opts()) -> + emqx_rpc:badrpc() | ok_or_error(emqx_node_rebalance_evacuation:start_error()). +node_rebalance_evacuation_start(Node, #{} = Opts) -> + rpc:call(Node, emqx_node_rebalance_evacuation, start, [Opts]). + +-spec node_rebalance_evacuation_stop(node()) -> + emqx_rpc:badrpc() | ok_or_error(not_started). +node_rebalance_evacuation_stop(Node) -> + rpc:call(Node, emqx_node_rebalance_evacuation, stop, []). + +-spec node_rebalance_start(node(), emqx_node_rebalance:start_opts()) -> + emqx_rpc:badrpc() | ok_or_error(emqx_node_rebalance:start_error()). +node_rebalance_start(Node, Opts) -> + rpc:call(Node, emqx_node_rebalance, start, [Opts]). + +-spec node_rebalance_stop(node()) -> + emqx_rpc:badrpc() | ok_or_error(not_started). +node_rebalance_stop(Node) -> + rpc:call(Node, emqx_node_rebalance, stop, []). diff --git a/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_evacuation_proto_v1.erl b/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_evacuation_proto_v1.erl new file mode 100644 index 000000000..f5a6e1077 --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_evacuation_proto_v1.erl @@ -0,0 +1,22 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_evacuation_proto_v1). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + available_nodes/1 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.22". + +-spec available_nodes([node()]) -> emqx_rpc:multicall_result(node()). +available_nodes(Nodes) -> + rpc:multicall(Nodes, emqx_node_rebalance_evacuation, is_node_available, []). diff --git a/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_proto_v1.erl b/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_proto_v1.erl new file mode 100644 index 000000000..98625d4fd --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_proto_v1.erl @@ -0,0 +1,62 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_proto_v1). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + available_nodes/1, + evict_connections/2, + evict_sessions/4, + connection_counts/1, + session_counts/1, + enable_rebalance_agent/2, + disable_rebalance_agent/2, + disconnected_session_counts/1 +]). + +-include_lib("emqx/include/bpapi.hrl"). +-include_lib("emqx/include/types.hrl"). + +introduced_in() -> + "5.0.22". + +-spec available_nodes([node()]) -> emqx_rpc:multicall_result(node()). +available_nodes(Nodes) -> + rpc:multicall(Nodes, emqx_node_rebalance, is_node_available, []). + +-spec evict_connections([node()], non_neg_integer()) -> + emqx_rpc:multicall_result(ok_or_error(disabled)). +evict_connections(Nodes, Count) -> + rpc:multicall(Nodes, emqx_eviction_agent, evict_connections, [Count]). + +-spec evict_sessions([node()], non_neg_integer(), [node()], emqx_channel:conn_state()) -> + emqx_rpc:multicall_result(ok_or_error(disabled)). +evict_sessions(Nodes, Count, RecipientNodes, ConnState) -> + rpc:multicall(Nodes, emqx_eviction_agent, evict_sessions, [Count, RecipientNodes, ConnState]). + +-spec connection_counts([node()]) -> emqx_rpc:multicall_result({ok, non_neg_integer()}). +connection_counts(Nodes) -> + rpc:multicall(Nodes, emqx_node_rebalance, connection_count, []). + +-spec session_counts([node()]) -> emqx_rpc:multicall_result({ok, non_neg_integer()}). +session_counts(Nodes) -> + rpc:multicall(Nodes, emqx_node_rebalance, session_count, []). + +-spec enable_rebalance_agent([node()], pid()) -> + emqx_rpc:multicall_result(ok_or_error(already_enabled | eviction_agent_busy)). +enable_rebalance_agent(Nodes, OwnerPid) -> + rpc:multicall(Nodes, emqx_node_rebalance_agent, enable, [OwnerPid]). + +-spec disable_rebalance_agent([node()], pid()) -> + emqx_rpc:multicall_result(ok_or_error(already_disabled | invalid_coordinator)). +disable_rebalance_agent(Nodes, OwnerPid) -> + rpc:multicall(Nodes, emqx_node_rebalance_agent, disable, [OwnerPid]). + +-spec disconnected_session_counts([node()]) -> emqx_rpc:multicall_result({ok, non_neg_integer()}). +disconnected_session_counts(Nodes) -> + rpc:multicall(Nodes, emqx_node_rebalance, disconnected_session_count, []). diff --git a/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_status_proto_v1.erl b/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_status_proto_v1.erl new file mode 100644 index 000000000..e3e4a423c --- /dev/null +++ b/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_status_proto_v1.erl @@ -0,0 +1,36 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_status_proto_v1). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + local_status/1, + rebalance_status/1, + evacuation_status/1 +]). + +-include_lib("emqx/include/bpapi.hrl"). +-include_lib("emqx/include/types.hrl"). + +introduced_in() -> + "5.0.22". + +-spec local_status(node()) -> + emqx_rpc:badrpc() | disabled | {evacuation, map()} | {rebalance, map()}. +local_status(Node) -> + rpc:call(Node, emqx_node_rebalance_status, local_status, []). + +-spec rebalance_status([node()]) -> + emqx_rpc:multicall_result({node(), map()}). +rebalance_status(Nodes) -> + rpc:multicall(Nodes, emqx_node_rebalance_status, rebalance_status, []). + +-spec evacuation_status([node()]) -> + emqx_rpc:multicall_result({node(), map()}). +evacuation_status(Nodes) -> + rpc:multicall(Nodes, emqx_node_rebalance_status, evacuation_status, []). diff --git a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl new file mode 100644 index 000000000..a818145a2 --- /dev/null +++ b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl @@ -0,0 +1,229 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/asserts.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-import( + emqx_eviction_agent_test_helpers, + [emqtt_connect_many/1, emqtt_connect_many/2, stop_many/1, case_specific_node_name/3] +). + +-define(START_APPS, [emqx_eviction_agent, emqx_node_rebalance]). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([]), + ok. + +init_per_testcase(Case, Config) -> + ClusterNodes = emqx_eviction_agent_test_helpers:start_cluster( + [ + {case_specific_node_name(?MODULE, Case, '_donor'), 2883}, + {case_specific_node_name(?MODULE, Case, '_recipient'), 3883} + ], + ?START_APPS + ), + ok = snabbkaffe:start_trace(), + [{cluster_nodes, ClusterNodes} | Config]. + +end_per_testcase(_Case, Config) -> + ok = snabbkaffe:stop(), + ok = emqx_eviction_agent_test_helpers:stop_cluster( + ?config(cluster_nodes, Config), + ?START_APPS + ). + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_rebalance(Config) -> + process_flag(trap_exit, true), + + [{DonorNode, DonorPort}, {RecipientNode, _RecipientPort}] = ?config(cluster_nodes, Config), + + Nodes = [DonorNode, RecipientNode], + + Conns = emqtt_connect_many(DonorPort, 500), + + Opts = #{ + conn_evict_rate => 10, + sess_evict_rate => 10, + evict_interval => 10, + abs_conn_threshold => 50, + abs_sess_threshold => 50, + rel_conn_threshold => 1.0, + rel_sess_threshold => 1.0, + wait_health_check => 0.01, + wait_takeover => 0.01, + nodes => Nodes + }, + + ?assertWaitEvent( + ok = rpc:call(DonorNode, emqx_node_rebalance, start, [Opts]), + #{?snk_kind := emqx_node_rebalance_evict_sess_over}, + 10000 + ), + + DonorConnCount = rpc:call(DonorNode, emqx_eviction_agent, connection_count, []), + DonorSessCount = rpc:call(DonorNode, emqx_eviction_agent, session_count, []), + DonorDSessCount = rpc:call(DonorNode, emqx_eviction_agent, session_count, [disconnected]), + + RecipientConnCount = rpc:call(RecipientNode, emqx_eviction_agent, connection_count, []), + RecipientSessCount = rpc:call(RecipientNode, emqx_eviction_agent, session_count, []), + RecipientDSessCount = rpc:call(RecipientNode, emqx_eviction_agent, session_count, [disconnected]), + + ct:pal( + "Donor: conn=~p, sess=~p, dsess=~p", + [DonorConnCount, DonorSessCount, DonorDSessCount] + ), + ct:pal( + "Recipient: conn=~p, sess=~p, dsess=~p", + [RecipientConnCount, RecipientSessCount, RecipientDSessCount] + ), + + ?assert(DonorConnCount - 50 =< RecipientConnCount), + ?assert(DonorDSessCount - 50 =< RecipientDSessCount), + + ok = stop_many(Conns). + +t_rebalance_node_crash(Config) -> + process_flag(trap_exit, true), + + [{DonorNode, DonorPort}, {RecipientNode, _RecipientPort}] = ?config(cluster_nodes, Config), + + Nodes = [DonorNode, RecipientNode], + + Conns = emqtt_connect_many(DonorPort, 500), + + Opts = #{ + conn_evict_rate => 10, + sess_evict_rate => 10, + evict_interval => 10, + abs_conn_threshold => 50, + abs_sess_threshold => 50, + rel_conn_threshold => 1.0, + rel_sess_threshold => 1.0, + wait_health_check => 0.01, + wait_takeover => 0.01, + nodes => Nodes + }, + + ?assertWaitEvent( + begin + ok = rpc:call(DonorNode, emqx_node_rebalance, start, [Opts]), + emqx_common_test_helpers:stop_slave(RecipientNode) + end, + #{?snk_kind := emqx_node_rebalance_started}, + 1000 + ), + + ?assertEqual( + disabled, + rpc:call(DonorNode, emqx_node_rebalance, status, []) + ), + + ok = stop_many(Conns). + +t_no_need_to_rebalance(Config) -> + process_flag(trap_exit, true), + + [{DonorNode, DonorPort}, {RecipientNode, _RecipientPort}] = ?config(cluster_nodes, Config), + + Nodes = [DonorNode, RecipientNode], + + Opts = #{ + conn_evict_rate => 10, + sess_evict_rate => 10, + evict_interval => 10, + abs_conn_threshold => 50, + abs_sess_threshold => 50, + rel_conn_threshold => 1.0, + rel_sess_threshold => 1.0, + wait_health_check => 0.01, + wait_takeover => 0.01, + nodes => Nodes + }, + + ?assertEqual( + {error, nothing_to_balance}, + rpc:call(DonorNode, emqx_node_rebalance, start, [Opts]) + ), + + Conns = emqtt_connect_many(DonorPort, 50), + + ?assertEqual( + {error, nothing_to_balance}, + rpc:call(DonorNode, emqx_node_rebalance, start, [Opts]) + ), + + ok = stop_many(Conns). + +t_unknown_mesages(Config) -> + process_flag(trap_exit, true), + [{DonorNode, DonorPort}, {RecipientNode, _RecipientPort}] = ?config(cluster_nodes, Config), + + Nodes = [DonorNode, RecipientNode], + + Conns = emqtt_connect_many(DonorPort, 500), + + Opts = #{ + wait_health_check => 100, + abs_conn_threshold => 50, + nodes => Nodes + }, + + Pid = rpc:call(DonorNode, erlang, whereis, [emqx_node_rebalance]), + + Pid ! unknown, + ok = gen_server:cast(Pid, unknown), + ?assertEqual( + ignored, + gen_server:call(Pid, unknown) + ), + + ok = rpc:call(DonorNode, emqx_node_rebalance, start, [Opts]), + + Pid ! unknown, + ok = gen_server:cast(Pid, unknown), + ?assertEqual( + ignored, + gen_server:call(Pid, unknown) + ), + + ok = stop_many(Conns). + +t_available_nodes(Config) -> + [{DonorNode, _DonorPort}, {RecipientNode, _RecipientPort}] = ?config(cluster_nodes, Config), + + %% Start eviction agent on RecipientNode so that it will be "occupied" + %% and not available for rebalance + ok = rpc:call(RecipientNode, emqx_eviction_agent, enable, [test_rebalance, undefined]), + + %% Only DonorNode should be is available for rebalance, since RecipientNode is "occupied" + ?assertEqual( + [DonorNode], + rpc:call( + DonorNode, + emqx_node_rebalance, + available_nodes, + [[DonorNode, RecipientNode]] + ) + ). diff --git a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl new file mode 100644 index 000000000..8b21f9433 --- /dev/null +++ b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl @@ -0,0 +1,214 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_agent_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-import( + emqx_eviction_agent_test_helpers, + [case_specific_node_name/2] +). + +all() -> + [ + {group, local}, + {group, cluster} + ]. + +groups() -> + [ + {local, [], [ + t_enable_disable, + t_enable_egent_busy, + t_unknown_messages + ]}, + {cluster, [], [ + t_rebalance_agent_coordinator_fail, + t_rebalance_agent_fail + ]} + ]. + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_eviction_agent, emqx_node_rebalance]), + ok. + +init_per_group(local, Config) -> + [{cluster, false} | Config]; +init_per_group(cluster, Config) -> + [{cluster, true} | Config]. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(Case, Config) -> + case ?config(cluster, Config) of + true -> + ClusterNodes = emqx_eviction_agent_test_helpers:start_cluster( + [{case_specific_node_name(?MODULE, Case), 2883}], + [emqx_eviction_agent, emqx_node_rebalance] + ), + [{cluster_nodes, ClusterNodes} | Config]; + false -> + Config + end. + +end_per_testcase(_Case, Config) -> + case ?config(cluster, Config) of + true -> + emqx_eviction_agent_test_helpers:stop_cluster( + ?config(cluster_nodes, Config), + [emqx_eviction_agent, emqx_node_rebalance] + ); + false -> + ok + end. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +%% Local tests + +t_enable_disable(_Config) -> + ?assertEqual( + disabled, + emqx_node_rebalance_agent:status() + ), + + ?assertEqual( + ok, + emqx_node_rebalance_agent:enable(self()) + ), + + ?assertEqual( + {error, already_enabled}, + emqx_node_rebalance_agent:enable(self()) + ), + + ?assertEqual( + {enabled, self()}, + emqx_node_rebalance_agent:status() + ), + + ?assertEqual( + {error, invalid_coordinator}, + emqx_node_rebalance_agent:disable(spawn_link(fun() -> ok end)) + ), + + ?assertEqual( + ok, + emqx_node_rebalance_agent:disable(self()) + ), + + ?assertEqual( + {error, already_disabled}, + emqx_node_rebalance_agent:disable(self()) + ), + + ?assertEqual( + disabled, + emqx_node_rebalance_agent:status() + ). + +t_enable_egent_busy(_Config) -> + ok = emqx_eviction_agent:enable(rebalance_test, undefined), + + ?assertEqual( + {error, eviction_agent_busy}, + emqx_node_rebalance_agent:enable(self()) + ), + + ok = emqx_eviction_agent:disable(rebalance_test). + +t_unknown_messages(_Config) -> + Pid = whereis(emqx_node_rebalance_agent), + + ok = gen_server:cast(Pid, unknown), + + Pid ! unknown, + + ignored = gen_server:call(Pid, unknown). + +%% Cluster tests + +% The following tests verify that emqx_node_rebalance_agent correctly links +% coordinator process with emqx_eviction_agent-s. + +t_rebalance_agent_coordinator_fail(Config) -> + process_flag(trap_exit, true), + + [{Node, _}] = ?config(cluster_nodes, Config), + + CoordinatorPid = spawn_link( + fun() -> + receive + done -> ok + end + end + ), + + ?assertEqual( + disabled, + rpc:call(Node, emqx_eviction_agent, status, []) + ), + + ?assertEqual( + ok, + rpc:call(Node, emqx_node_rebalance_agent, enable, [CoordinatorPid]) + ), + + ?assertMatch( + {enabled, _}, + rpc:call(Node, emqx_eviction_agent, status, []) + ), + + EvictionAgentPid = rpc:call(Node, erlang, whereis, [emqx_eviction_agent]), + true = link(EvictionAgentPid), + + true = exit(CoordinatorPid, kill), + + receive + {'EXIT', EvictionAgentPid, _} -> true + after 1000 -> + ct:fail("emqx_eviction_agent did not exit") + end. + +t_rebalance_agent_fail(Config) -> + process_flag(trap_exit, true), + + [{Node, _}] = ?config(cluster_nodes, Config), + + CoordinatorPid = spawn_link( + fun() -> + receive + done -> ok + end + end + ), + + ?assertEqual( + ok, + rpc:call(Node, emqx_node_rebalance_agent, enable, [CoordinatorPid]) + ), + + EvictionAgentPid = rpc:call(Node, erlang, whereis, [emqx_eviction_agent]), + true = exit(EvictionAgentPid, kill), + + receive + {'EXIT', CoordinatorPid, _} -> true + after 1000 -> + ct:fail("emqx_node_rebalance_agent did not exit") + end. diff --git a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl new file mode 100644 index 000000000..21608b8bc --- /dev/null +++ b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl @@ -0,0 +1,444 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-import( + emqx_mgmt_api_test_util, + [ + request/2, + request/3, + uri/1 + ] +). + +-import( + emqx_eviction_agent_test_helpers, + [emqtt_connect_many/2, stop_many/1, case_specific_node_name/3] +). + +-define(START_APPS, [emqx_eviction_agent, emqx_node_rebalance]). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps(?START_APPS), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps(?START_APPS), + ok. + +init_per_testcase(Case, Config) -> + [{DonorNode, _} | _] = + ClusterNodes = emqx_eviction_agent_test_helpers:start_cluster( + [ + {case_specific_node_name(?MODULE, Case, '_donor'), 2883}, + {case_specific_node_name(?MODULE, Case, '_recipient'), 3883} + ], + ?START_APPS, + [{emqx, data_dir, case_specific_data_dir(Case, Config)}] + ), + + ok = rpc:call(DonorNode, emqx_mgmt_api_test_util, init_suite, []), + ok = take_auth_header_from(DonorNode), + + [{cluster_nodes, ClusterNodes} | Config]. +end_per_testcase(_Case, Config) -> + _ = emqx_eviction_agent_test_helpers:stop_cluster( + ?config(cluster_nodes, Config), + ?START_APPS + ). + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_start_evacuation_validation(Config) -> + [{DonorNode, _}, {RecipientNode, _}] = ?config(cluster_nodes, Config), + BadOpts = [ + #{conn_evict_rate => <<"conn">>}, + #{sess_evict_rate => <<"sess">>}, + #{redirect_to => 123}, + #{wait_takeover => <<"wait">>}, + #{migrate_to => []}, + #{migrate_to => <<"migrate_to">>}, + #{migrate_to => [<<"bad_node">>]}, + #{migrate_to => [<<"bad_node">>, atom_to_binary(DonorNode)]}, + #{unknown => <<"Value">>} + ], + lists:foreach( + fun(Opts) -> + ?assertMatch( + {ok, 400, #{}}, + api_post( + ["load_rebalance", atom_to_list(DonorNode), "evacuation", "start"], + Opts + ) + ) + end, + BadOpts + ), + ?assertMatch( + {ok, 400, #{}}, + api_post( + ["load_rebalance", "bad@node", "evacuation", "start"], + #{} + ) + ), + + ?assertMatch( + {ok, 200, #{}}, + api_post( + ["load_rebalance", atom_to_list(DonorNode), "evacuation", "start"], + #{ + conn_evict_rate => 10, + sess_evict_rate => 10, + wait_takeover => 10, + redirect_to => <<"srv">>, + migrate_to => [atom_to_binary(RecipientNode)] + } + ) + ), + + DonorNodeBin = atom_to_binary(DonorNode), + ?assertMatch( + {ok, 200, #{<<"evacuations">> := [#{<<"node">> := DonorNodeBin}]}}, + api_get(["load_rebalance", "global_status"]) + ). + +t_start_rebalance_validation(Config) -> + process_flag(trap_exit, true), + + [{DonorNode, DonorPort}, {RecipientNode, _}] = ?config(cluster_nodes, Config), + + BadOpts = [ + #{conn_evict_rate => <<"conn">>}, + #{sess_evict_rate => <<"sess">>}, + #{abs_conn_threshold => <<"act">>}, + #{rel_conn_threshold => <<"rct">>}, + #{abs_sess_threshold => <<"act">>}, + #{rel_sess_threshold => <<"rct">>}, + #{wait_takeover => <<"wait">>}, + #{wait_health_check => <<"wait">>}, + #{nodes => <<"nodes">>}, + #{nodes => []}, + #{nodes => [<<"bad_node">>]}, + #{nodes => [<<"bad_node">>, atom_to_binary(DonorNode)]}, + #{unknown => <<"Value">>} + ], + lists:foreach( + fun(Opts) -> + ?assertMatch( + {ok, 400, #{}}, + api_post( + ["load_rebalance", atom_to_list(DonorNode), "start"], + Opts + ) + ) + end, + BadOpts + ), + ?assertMatch( + {ok, 400, #{}}, + api_post( + ["load_rebalance", "bad@node", "start"], + #{} + ) + ), + + Conns = emqtt_connect_many(DonorPort, 50), + + ?assertMatch( + {ok, 200, #{}}, + api_post( + ["load_rebalance", atom_to_list(DonorNode), "start"], + #{ + conn_evict_rate => 10, + sess_evict_rate => 10, + wait_takeover => 10, + wait_health_check => 10, + abs_conn_threshold => 10, + rel_conn_threshold => 1.001, + abs_sess_threshold => 10, + rel_sess_threshold => 1.001, + nodes => [ + atom_to_binary(DonorNode), + atom_to_binary(RecipientNode) + ] + } + ) + ), + + DonorNodeBin = atom_to_binary(DonorNode), + ?assertMatch( + {ok, 200, #{<<"rebalances">> := [#{<<"node">> := DonorNodeBin}]}}, + api_get(["load_rebalance", "global_status"]) + ), + + ok = stop_many(Conns). + +t_start_stop_evacuation(Config) -> + [{DonorNode, _}, {RecipientNode, _}] = ?config(cluster_nodes, Config), + + StartOpts = maps:merge( + emqx_node_rebalance_api:rebalance_evacuation_example(), + #{migrate_to => [atom_to_binary(RecipientNode)]} + ), + + ?assertMatch( + {ok, 200, #{}}, + api_post( + ["load_rebalance", atom_to_list(DonorNode), "evacuation", "start"], + StartOpts + ) + ), + + StatusResponse = api_get(["load_rebalance", "status"]), + + ?assertMatch( + {ok, 200, _}, + StatusResponse + ), + + {ok, 200, Status} = StatusResponse, + + ?assertMatch( + #{ + process := evacuation, + connection_eviction_rate := 100, + session_eviction_rate := 100, + connection_goal := 0, + session_goal := 0, + stats := #{ + initial_connected := _, + current_connected := _, + initial_sessions := _, + current_sessions := _ + } + }, + emqx_node_rebalance_api:translate(local_status_enabled, Status) + ), + + DonorNodeBin = atom_to_binary(DonorNode), + + GlobalStatusResponse = api_get(["load_rebalance", "global_status"]), + + ?assertMatch( + {ok, 200, _}, + GlobalStatusResponse + ), + + {ok, 200, GlobalStatus} = GlobalStatusResponse, + + ?assertMatch( + #{ + rebalances := [], + evacuations := [ + #{ + node := DonorNodeBin, + connection_eviction_rate := 100, + session_eviction_rate := 100, + connection_goal := 0, + session_goal := 0, + stats := #{ + initial_connected := _, + current_connected := _, + initial_sessions := _, + current_sessions := _ + } + } + ] + }, + emqx_node_rebalance_api:translate(global_status, GlobalStatus) + ), + + ?assertMatch( + {ok, 200, #{}}, + api_post( + ["load_rebalance", atom_to_list(DonorNode), "evacuation", "stop"], + #{} + ) + ), + + ?assertMatch( + {ok, 200, #{<<"status">> := <<"disabled">>}}, + api_get(["load_rebalance", "status"]) + ), + + ?assertMatch( + {ok, 200, #{<<"evacuations">> := [], <<"rebalances">> := []}}, + api_get(["load_rebalance", "global_status"]) + ). + +t_start_stop_rebalance(Config) -> + process_flag(trap_exit, true), + + [{DonorNode, DonorPort}, {RecipientNode, _}] = ?config(cluster_nodes, Config), + + ?assertMatch( + {ok, 200, #{<<"status">> := <<"disabled">>}}, + api_get(["load_rebalance", "status"]) + ), + + Conns = emqtt_connect_many(DonorPort, 100), + + StartOpts = maps:without( + [nodes], + emqx_node_rebalance_api:rebalance_example() + ), + + ?assertMatch( + {ok, 200, #{}}, + api_post( + ["load_rebalance", atom_to_list(DonorNode), "start"], + StartOpts + ) + ), + + StatusResponse = api_get(["load_rebalance", "status"]), + + ?assertMatch( + {ok, 200, _}, + StatusResponse + ), + + {ok, 200, Status} = StatusResponse, + + ?assertMatch( + #{process := rebalance, connection_eviction_rate := 10, session_eviction_rate := 20}, + emqx_node_rebalance_api:translate(local_status_enabled, Status) + ), + + DonorNodeBin = atom_to_binary(DonorNode), + RecipientNodeBin = atom_to_binary(RecipientNode), + + GlobalStatusResponse = api_get(["load_rebalance", "global_status"]), + + ?assertMatch( + {ok, 200, _}, + GlobalStatusResponse + ), + + {ok, 200, GlobalStatus} = GlobalStatusResponse, + + ?assertMatch( + {ok, 200, #{ + <<"evacuations">> := [], + <<"rebalances">> := + [ + #{ + <<"state">> := _, + <<"node">> := DonorNodeBin, + <<"coordinator_node">> := _, + <<"connection_eviction_rate">> := 10, + <<"session_eviction_rate">> := 20, + <<"donors">> := [DonorNodeBin], + <<"recipients">> := [RecipientNodeBin] + } + ] + }}, + api_get(["load_rebalance", "global_status"]) + ), + + ?assertMatch( + #{ + evacuations := [], + rebalances := [ + #{ + state := _, + node := DonorNodeBin, + coordinator_node := _, + connection_eviction_rate := 10, + session_eviction_rate := 20, + donors := [DonorNodeBin], + recipients := [RecipientNodeBin] + } + ] + }, + emqx_node_rebalance_api:translate(global_status, GlobalStatus) + ), + + ?assertMatch( + {ok, 200, #{}}, + api_post( + ["load_rebalance", atom_to_list(DonorNode), "stop"], + #{} + ) + ), + + ?assertMatch( + {ok, 200, #{<<"status">> := <<"disabled">>}}, + api_get(["load_rebalance", "status"]) + ), + + ?assertMatch( + {ok, 200, #{<<"evacuations">> := [], <<"rebalances">> := []}}, + api_get(["load_rebalance", "global_status"]) + ), + + ok = stop_many(Conns). + +t_availability_check(Config) -> + [{DonorNode, _} | _] = ?config(cluster_nodes, Config), + ?assertMatch( + {ok, 200, #{}}, + api_get(["load_rebalance", "availability_check"]) + ), + + ok = rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [#{}]), + + ?assertMatch( + {ok, 503, _}, + api_get(["load_rebalance", "availability_check"]) + ), + + ok = rpc:call(DonorNode, emqx_node_rebalance_evacuation, stop, []), + + ?assertMatch( + {ok, 200, #{}}, + api_get(["load_rebalance", "availability_check"]) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +api_get(Path) -> + case request(get, uri(Path)) of + {ok, Code, ResponseBody} -> + {ok, Code, jiffy:decode(ResponseBody, [return_maps])}; + {error, _} = Error -> + Error + end. + +api_post(Path, Data) -> + case request(post, uri(Path), Data) of + {ok, Code, ResponseBody} -> + {ok, Code, jiffy:decode(ResponseBody, [return_maps])}; + {error, _} = Error -> + Error + end. + +take_auth_header_from(Node) -> + meck:new(emqx_common_test_http, [passthrough]), + meck:expect( + emqx_common_test_http, + default_auth_header, + fun() -> rpc:call(Node, emqx_common_test_http, default_auth_header, []) end + ), + ok. + +case_specific_data_dir(Case, Config) -> + case ?config(priv_dir, Config) of + undefined -> undefined; + PrivDir -> filename:join(PrivDir, atom_to_list(Case)) + end. diff --git a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl new file mode 100644 index 000000000..54ecad026 --- /dev/null +++ b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl @@ -0,0 +1,291 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_cli_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-import( + emqx_eviction_agent_test_helpers, + [emqtt_connect_many/2, stop_many/1, case_specific_node_name/3] +). + +-define(START_APPS, [emqx_eviction_agent, emqx_node_rebalance]). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_common_test_helpers:start_apps(?START_APPS), + Config. + +end_per_suite(Config) -> + emqx_common_test_helpers:stop_apps(lists:reverse(?START_APPS)), + Config. + +init_per_testcase(Case = t_rebalance, Config) -> + _ = emqx_node_rebalance_evacuation:stop(), + ClusterNodes = emqx_eviction_agent_test_helpers:start_cluster( + [ + {case_specific_node_name(?MODULE, Case, '_donor'), 2883}, + {case_specific_node_name(?MODULE, Case, '_recipient'), 3883} + ], + ?START_APPS + ), + [{cluster_nodes, ClusterNodes} | Config]; +init_per_testcase(_Case, Config) -> + _ = emqx_node_rebalance_evacuation:stop(), + _ = emqx_node_rebalance:stop(), + Config. + +end_per_testcase(t_rebalance, Config) -> + _ = emqx_node_rebalance_evacuation:stop(), + _ = emqx_node_rebalance:stop(), + _ = emqx_eviction_agent_test_helpers:stop_cluster( + ?config(cluster_nodes, Config), + ?START_APPS + ); +end_per_testcase(_Case, _Config) -> + _ = emqx_node_rebalance_evacuation:stop(), + _ = emqx_node_rebalance:stop(). + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_evacuation(_Config) -> + %% usage + ok = emqx_node_rebalance_cli:cli(["foobar"]), + + %% status + ok = emqx_node_rebalance_cli:cli(["status"]), + ok = emqx_node_rebalance_cli:cli(["node-status"]), + ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]), + + %% start with invalid args + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--evacuation", "--foo-bar"]) + ), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--evacuation", "--conn-evict-rate", "foobar"]) + ), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--evacuation", "--sess-evict-rate", "foobar"]) + ), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--evacuation", "--wait-takeover", "foobar"]) + ), + + ?assertNot( + emqx_node_rebalance_cli:cli([ + "start", + "--evacuation", + "--migrate-to", + "nonexistent@node" + ]) + ), + ?assertNot( + emqx_node_rebalance_cli:cli([ + "start", + "--evacuation", + "--migrate-to", + "" + ]) + ), + ?assertNot( + emqx_node_rebalance_cli:cli([ + "start", + "--evacuation", + "--unknown-arg" + ]) + ), + ?assert( + emqx_node_rebalance_cli:cli([ + "start", + "--evacuation", + "--conn-evict-rate", + "10", + "--sess-evict-rate", + "10", + "--wait-takeover", + "10", + "--migrate-to", + atom_to_list(node()), + "--redirect-to", + "srv" + ]) + ), + + %% status + ok = emqx_node_rebalance_cli:cli(["status"]), + ok = emqx_node_rebalance_cli:cli(["node-status"]), + ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]), + + ?assertMatch( + {enabled, #{}}, + emqx_node_rebalance_evacuation:status() + ), + + %% already enabled + ?assertNot( + emqx_node_rebalance_cli:cli([ + "start", + "--evacuation", + "--conn-evict-rate", + "10", + "--redirect-to", + "srv" + ]) + ), + + %% stop + true = emqx_node_rebalance_cli:cli(["stop"]), + + false = emqx_node_rebalance_cli:cli(["stop"]), + + ?assertEqual( + disabled, + emqx_node_rebalance_evacuation:status() + ). + +t_rebalance(Config) -> + process_flag(trap_exit, true), + + [{DonorNode, DonorPort}, {RecipientNode, _}] = ?config(cluster_nodes, Config), + + %% start with invalid args + ?assertNot( + emqx_node_rebalance_cli(DonorNode, ["start", "--foo-bar"]) + ), + + ?assertNot( + emqx_node_rebalance_cli(DonorNode, ["start", "--conn-evict-rate", "foobar"]) + ), + + ?assertNot( + emqx_node_rebalance_cli(DonorNode, ["start", "--abs-conn-threshold", "foobar"]) + ), + + ?assertNot( + emqx_node_rebalance_cli(DonorNode, ["start", "--rel-conn-threshold", "foobar"]) + ), + + ?assertNot( + emqx_node_rebalance_cli(DonorNode, ["start", "--sess-evict-rate", "foobar"]) + ), + + ?assertNot( + emqx_node_rebalance_cli(DonorNode, ["start", "--abs-sess-threshold", "foobar"]) + ), + + ?assertNot( + emqx_node_rebalance_cli(DonorNode, ["start", "--rel-sess-threshold", "foobar"]) + ), + + ?assertNot( + emqx_node_rebalance_cli(DonorNode, ["start", "--wait-takeover", "foobar"]) + ), + + ?assertNot( + emqx_node_rebalance_cli(DonorNode, ["start", "--wait-health-check", "foobar"]) + ), + + ?assertNot( + emqx_node_rebalance_cli(DonorNode, [ + "start", + "--nodes", + "nonexistent@node" + ]) + ), + ?assertNot( + emqx_node_rebalance_cli(DonorNode, [ + "start", + "--nodes", + "" + ]) + ), + ?assertNot( + emqx_node_rebalance_cli(DonorNode, [ + "start", + "--nodes", + atom_to_list(RecipientNode) + ]) + ), + ?assertNot( + emqx_node_rebalance_cli(DonorNode, [ + "start", + "--unknown-arg" + ]) + ), + + Conns = emqtt_connect_many(DonorPort, 20), + + ?assert( + emqx_node_rebalance_cli(DonorNode, [ + "start", + "--conn-evict-rate", + "10", + "--abs-conn-threshold", + "10", + "--rel-conn-threshold", + "1.1", + "--sess-evict-rate", + "10", + "--abs-sess-threshold", + "10", + "--rel-sess-threshold", + "1.1", + "--wait-takeover", + "10", + "--nodes", + atom_to_list(DonorNode) ++ "," ++ + atom_to_list(RecipientNode) + ]) + ), + + %% status + ok = emqx_node_rebalance_cli(DonorNode, ["status"]), + ok = emqx_node_rebalance_cli(DonorNode, ["node-status"]), + ok = emqx_node_rebalance_cli(DonorNode, ["node-status", atom_to_list(DonorNode)]), + + ?assertMatch( + {enabled, #{}}, + rpc:call(DonorNode, emqx_node_rebalance, status, []) + ), + + %% already enabled + ?assertNot( + emqx_node_rebalance_cli(DonorNode, ["start"]) + ), + + %% stop + true = emqx_node_rebalance_cli(DonorNode, ["stop"]), + + false = emqx_node_rebalance_cli(DonorNode, ["stop"]), + + ?assertEqual( + disabled, + rpc:call(DonorNode, emqx_node_rebalance, status, []) + ), + + ok = stop_many(Conns). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +emqx_node_rebalance_cli(Node, Args) -> + case rpc:call(Node, emqx_node_rebalance_cli, cli, [Args]) of + {badrpc, Reason} -> + error(Reason); + Result -> + Result + end. diff --git a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl new file mode 100644 index 000000000..cdafad97a --- /dev/null +++ b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl @@ -0,0 +1,271 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_evacuation_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/asserts.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-import( + emqx_eviction_agent_test_helpers, + [emqtt_connect/1, emqtt_try_connect/1, case_specific_node_name/3] +). + +all() -> [{group, one_node}, {group, two_node}]. + +groups() -> + [ + {one_node, [], [ + t_agent_busy, + t_already_started, + t_not_started, + t_start, + t_persistence, + t_unknown_messages + ]}, + {two_node, [], [ + t_conn_evicted, + t_migrate_to, + t_session_evicted + ]} + ]. + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([]), + ok. + +init_per_group(one_node, Config) -> + [{cluster_type, one_node} | Config]; +init_per_group(two_node, Config) -> + [{cluster_type, two_node} | Config]. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(Case, Config) -> + NodesWithPorts = + case ?config(cluster_type, Config) of + one_node -> + [{case_specific_node_name(?MODULE, Case, '_evacuated'), 2883}]; + two_node -> + [ + {case_specific_node_name(?MODULE, Case, '_evacuated'), 2883}, + {case_specific_node_name(?MODULE, Case, '_recipient'), 3883} + ] + end, + ClusterNodes = emqx_eviction_agent_test_helpers:start_cluster( + NodesWithPorts, + [emqx_eviction_agent, emqx_node_rebalance], + [{emqx, data_dir, case_specific_data_dir(Case, Config)}] + ), + ok = snabbkaffe:start_trace(), + [{cluster_nodes, ClusterNodes} | Config]. + +end_per_testcase(_Case, Config) -> + ok = snabbkaffe:stop(), + ok = emqx_eviction_agent_test_helpers:stop_cluster( + ?config(cluster_nodes, Config), + [emqx_eviction_agent, emqx_node_rebalance] + ). + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +%% One node tests + +t_agent_busy(Config) -> + [{DonorNode, _DonorPort}] = ?config(cluster_nodes, Config), + ok = rpc:call(DonorNode, emqx_eviction_agent, enable, [other_rebalance, undefined]), + + ?assertEqual( + {error, eviction_agent_busy}, + rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)]) + ). + +t_already_started(Config) -> + [{DonorNode, _DonorPort}] = ?config(cluster_nodes, Config), + ok = rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)]), + + ?assertEqual( + {error, already_started}, + rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)]) + ). + +t_not_started(Config) -> + [{DonorNode, _DonorPort}] = ?config(cluster_nodes, Config), + + ?assertEqual( + {error, not_started}, + rpc:call(DonorNode, emqx_node_rebalance_evacuation, stop, []) + ). + +t_start(Config) -> + process_flag(trap_exit, true), + + [{DonorNode, DonorPort}] = ?config(cluster_nodes, Config), + + ok = rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)]), + ?assertMatch( + {error, {use_another_server, #{}}}, + emqtt_try_connect([{port, DonorPort}]) + ). + +t_persistence(Config) -> + process_flag(trap_exit, true), + + [{DonorNode, DonorPort}] = ?config(cluster_nodes, Config), + + ok = rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)]), + + ?assertMatch( + {error, {use_another_server, #{}}}, + emqtt_try_connect([{port, DonorPort}]) + ), + + ok = rpc:call(DonorNode, supervisor, terminate_child, [ + emqx_node_rebalance_sup, emqx_node_rebalance_evacuation + ]), + {ok, _} = rpc:call(DonorNode, supervisor, restart_child, [ + emqx_node_rebalance_sup, emqx_node_rebalance_evacuation + ]), + + ?assertMatch( + {error, {use_another_server, #{}}}, + emqtt_try_connect([{port, DonorPort}]) + ), + ?assertMatch( + {enabled, #{conn_evict_rate := 10}}, + rpc:call(DonorNode, emqx_node_rebalance_evacuation, status, []) + ). + +t_unknown_messages(Config) -> + process_flag(trap_exit, true), + + [{DonorNode, _DonorPort}] = ?config(cluster_nodes, Config), + + ok = rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)]), + + Pid = rpc:call(DonorNode, erlang, whereis, [emqx_node_rebalance_evacuation]), + + Pid ! unknown, + + ok = gen_server:cast(Pid, unknown), + + ?assertEqual( + ignored, + gen_server:call(Pid, unknown) + ). + +%% Two node tests + +t_conn_evicted(Config) -> + process_flag(trap_exit, true), + + [{DonorNode, DonorPort}, _] = ?config(cluster_nodes, Config), + + {ok, C} = emqtt_connect([{clientid, <<"evacuated">>}, {port, DonorPort}]), + + ?assertWaitEvent( + ok = rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)]), + #{?snk_kind := node_evacuation_evict_conn}, + 1000 + ), + + ?assertMatch( + {error, {use_another_server, #{}}}, + emqtt_try_connect([{clientid, <<"connecting">>}, {port, DonorPort}]) + ), + + receive + {'EXIT', C, {disconnected, 156, _}} -> ok + after 1000 -> + ct:fail("Connection not evicted") + end. + +t_migrate_to(Config) -> + [{DonorNode, _DonorPort}, {RecipientNode, _RecipientPort}] = ?config(cluster_nodes, Config), + + ?assertEqual( + [RecipientNode], + rpc:call(DonorNode, emqx_node_rebalance_evacuation, migrate_to, [undefined]) + ), + + ?assertEqual( + [], + rpc:call(DonorNode, emqx_node_rebalance_evacuation, migrate_to, [['unknown@node']]) + ), + + ok = rpc:call(RecipientNode, emqx_eviction_agent, enable, [test_rebalance, undefined]), + + ?assertEqual( + [], + rpc:call(DonorNode, emqx_node_rebalance_evacuation, migrate_to, [undefined]) + ). + +t_session_evicted(Config) -> + process_flag(trap_exit, true), + + [{DonorNode, DonorPort}, {RecipientNode, _RecipientPort}] = ?config(cluster_nodes, Config), + + {ok, C} = emqtt_connect([ + {port, DonorPort}, {clientid, <<"client_with_sess">>}, {clean_start, false} + ]), + + ?assertWaitEvent( + ok = rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)]), + #{?snk_kind := node_evacuation_evict_sess_over}, + 5000 + ), + + receive + {'EXIT', C, {disconnected, ?RC_USE_ANOTHER_SERVER, _}} -> ok + after 1000 -> + ct:fail("Connection not evicted") + end, + + [ChannelPid] = rpc:call(DonorNode, emqx_cm_registry, lookup_channels, [<<"client_with_sess">>]), + + ?assertEqual( + RecipientNode, + node(ChannelPid) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +opts(Config) -> + #{ + server_reference => <<"srv">>, + conn_evict_rate => 10, + sess_evict_rate => 10, + wait_takeover => 1, + migrate_to => migrate_to(Config) + }. + +migrate_to(Config) -> + case ?config(cluster_type, Config) of + one_node -> + []; + two_node -> + [_, {RecipientNode, _RecipientPort}] = ?config(cluster_nodes, Config), + [RecipientNode] + end. + +case_specific_data_dir(Case, Config) -> + case ?config(priv_dir, Config) of + undefined -> undefined; + PrivDir -> filename:join(PrivDir, atom_to_list(Case)) + end. diff --git a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl new file mode 100644 index 000000000..450280cb8 --- /dev/null +++ b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl @@ -0,0 +1,108 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_evacuation_persist_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(_Case, Config) -> + _ = emqx_node_rebalance_evacuation_persist:clear(), + Config. + +end_per_testcase(_Case, _Config) -> + _ = emqx_node_rebalance_evacuation_persist:clear(). + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_save_read(_Config) -> + DefaultOpts = #{ + server_reference => <<"default_ref">>, + conn_evict_rate => 2001, + sess_evict_rate => 2002, + wait_takeover => 2003 + }, + + Opts0 = #{ + server_reference => <<"ref">>, + conn_evict_rate => 1001, + sess_evict_rate => 1002, + wait_takeover => 1003 + }, + ok = emqx_node_rebalance_evacuation_persist:save(Opts0), + + {ok, ReadOpts0} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts), + ?assertEqual(Opts0, ReadOpts0), + + Opts1 = Opts0#{server_reference => undefined}, + ok = emqx_node_rebalance_evacuation_persist:save(Opts1), + + {ok, ReadOpts1} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts), + ?assertEqual(Opts1, ReadOpts1). + +t_read_default(_Config) -> + ok = write_evacuation_file(<<"{}">>), + + DefaultOpts = #{ + server_reference => <<"ref">>, + conn_evict_rate => 1001, + sess_evict_rate => 1002, + wait_takeover => 1003 + }, + + {ok, ReadOpts} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts), + ?assertEqual(DefaultOpts, ReadOpts). + +t_read_bad_data(_Config) -> + ok = write_evacuation_file(<<"{bad json">>), + + DefaultOpts = #{ + server_reference => <<"ref">>, + conn_evict_rate => 1001, + sess_evict_rate => 1002, + wait_takeover => 1003 + }, + + {ok, ReadOpts} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts), + ?assertEqual(DefaultOpts, ReadOpts). + +t_clear(_Config) -> + ok = write_evacuation_file(<<"{}">>), + + ?assertMatch( + {ok, _}, + emqx_node_rebalance_evacuation_persist:read(#{}) + ), + + ok = emqx_node_rebalance_evacuation_persist:clear(), + + ?assertEqual( + none, + emqx_node_rebalance_evacuation_persist:read(#{}) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +write_evacuation_file(Json) -> + ok = filelib:ensure_dir(emqx_node_rebalance_evacuation_persist:evacuation_filepath()), + ok = file:write_file( + emqx_node_rebalance_evacuation_persist:evacuation_filepath(), + Json + ). diff --git a/mix.exs b/mix.exs index 564d81ccf..cafeec7bd 100644 --- a/mix.exs +++ b/mix.exs @@ -402,7 +402,9 @@ defmodule EMQXUmbrella.MixProject do emqx_oracle: :permanent, emqx_bridge_oracle: :permanent, emqx_bridge_rabbitmq: :permanent, - emqx_ee_schema_registry: :permanent + emqx_ee_schema_registry: :permanent, + emqx_eviction_agent: :permanent, + emqx_node_rebalance: :permanent ], else: [] ) diff --git a/rebar.config.erl b/rebar.config.erl index a48a365c9..d556b41aa 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -478,7 +478,9 @@ relx_apps_per_edition(ee) -> emqx_oracle, emqx_bridge_oracle, emqx_bridge_rabbitmq, - emqx_ee_schema_registry + emqx_ee_schema_registry, + emqx_eviction_agent, + emqx_node_rebalance ]; relx_apps_per_edition(ce) -> []. From e683d28973c5aa9e2e52bcf19b9b2158657e92f3 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 4 May 2023 23:33:46 +0300 Subject: [PATCH 041/144] chore(rebalance): rebase and review fixes --- apps/emqx/src/emqx_cm.erl | 6 +- apps/emqx_utils/src/emqx_utils_api.erl | 4 +- .../i18n/emqx_eviction_agent_api_i18n.conf | 14 - .../src/emqx_eviction_agent.erl | 4 +- .../src/emqx_eviction_agent_app.erl | 2 - .../src/emqx_eviction_agent_channel.erl | 20 +- .../test/emqx_eviction_agent_SUITE.erl | 70 ++- .../test/emqx_eviction_agent_test_helpers.erl | 11 +- .../i18n/emqx_node_rebalance_api_i18n.conf | 490 ------------------ .../src/emqx_node_rebalance_api.erl | 41 +- .../src/emqx_node_rebalance_cli.erl | 4 +- ...emqx_node_rebalance_evacuation_persist.erl | 4 +- .../src/emqx_node_rebalance_status.erl | 2 +- .../test/emqx_node_rebalance_api_SUITE.erl | 6 +- .../emqx_node_rebalance_evacuation_SUITE.erl | 25 +- rel/i18n/emqx_eviction_agent_api.hocon | 9 + rel/i18n/emqx_node_rebalance_api.hocon | 267 ++++++++++ rel/i18n/zh/emqx_eviction_agent_api.hocon | 9 + rel/i18n/zh/emqx_node_rebalance_api.hocon | 267 ++++++++++ 19 files changed, 673 insertions(+), 582 deletions(-) delete mode 100644 lib-ee/emqx_eviction_agent/i18n/emqx_eviction_agent_api_i18n.conf delete mode 100644 lib-ee/emqx_node_rebalance/i18n/emqx_node_rebalance_api_i18n.conf create mode 100644 rel/i18n/emqx_eviction_agent_api.hocon create mode 100644 rel/i18n/emqx_node_rebalance_api.hocon create mode 100644 rel/i18n/zh/emqx_eviction_agent_api.hocon create mode 100644 rel/i18n/zh/emqx_node_rebalance_api.hocon diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index c8296f317..66c1db36e 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -602,14 +602,14 @@ all_channels() -> ets:select(?CHAN_TAB, Pat). %% @doc Get clientinfo for all clients with sessions -channel_with_session_table(ConnModules) -> +channel_with_session_table(ConnModuleList) -> Ms = ets:fun2ms( fun({{ClientId, _ChanPid}, Info, _Stats}) -> {ClientId, Info} end ), Table = ets:table(?CHAN_INFO_TAB, [{traverse, {select, Ms}}]), - ConnModuleMap = maps:from_list([{Mod, true} || Mod <- ConnModules]), + ConnModules = sets:from_list(ConnModuleList, [{version, 2}]), qlc:q([ {ClientId, ConnState, ConnInfo, ClientInfo} || {ClientId, #{ @@ -618,7 +618,7 @@ channel_with_session_table(ConnModules) -> conninfo := #{clean_start := false, conn_mod := ConnModule} = ConnInfo }} <- Table, - maps:is_key(ConnModule, ConnModuleMap) + sets:is_element(ConnModule, ConnModules) ]). %% @doc Get all local connection query handle diff --git a/apps/emqx_utils/src/emqx_utils_api.erl b/apps/emqx_utils/src/emqx_utils_api.erl index e6bd07272..a1bc97cd6 100644 --- a/apps/emqx_utils/src/emqx_utils_api.erl +++ b/apps/emqx_utils/src/emqx_utils_api.erl @@ -72,4 +72,6 @@ is_running_node(Node) -> handle_result({ok, Result}) -> ?OK(Result); handle_result({error, Reason}) -> - ?BAD_REQUEST(Reason). + ?BAD_REQUEST(Reason); +handle_result({HTTPCode, Content}) when is_integer(HTTPCode) -> + {HTTPCode, Content}. diff --git a/lib-ee/emqx_eviction_agent/i18n/emqx_eviction_agent_api_i18n.conf b/lib-ee/emqx_eviction_agent/i18n/emqx_eviction_agent_api_i18n.conf deleted file mode 100644 index 8bb7282c3..000000000 --- a/lib-ee/emqx_eviction_agent/i18n/emqx_eviction_agent_api_i18n.conf +++ /dev/null @@ -1,14 +0,0 @@ -emqx_eviction_agent_api { - - node_eviction_status_get { - desc { - en: "Get the node eviction status" - zh: "获取节点驱逐状态" - } - label { - en: "Node Eviction Status" - zh: "节点驱逐状态" - } - } - -} diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.erl b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.erl index b8e1b5236..9a29adc69 100644 --- a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.erl +++ b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.erl @@ -48,7 +48,9 @@ -export_type([server_reference/0]). --define(CONN_MODULES, [emqx_connection, emqx_ws_connection, emqx_eviction_agent_channel]). +-define(CONN_MODULES, [ + emqx_connection, emqx_ws_connection, emqx_quic_connection, emqx_eviction_agent_channel +]). %%-------------------------------------------------------------------- %% APIs diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_app.erl b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_app.erl index 63af59b09..90b09884f 100644 --- a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_app.erl +++ b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_app.erl @@ -6,8 +6,6 @@ -behaviour(application). --emqx_plugin(?MODULE). - -export([ start/2, stop/1 diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl index a42033c0f..a6097f03d 100644 --- a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl +++ b/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl @@ -13,8 +13,6 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --logger_header("[Evicted Channel]"). - -export([ start_link/1, start_supervised/1, @@ -33,13 +31,6 @@ code_change/3 ]). --import( - emqx_misc, - [ - maybe_apply/2 - ] -). - -type opts() :: #{ conninfo := emqx_types:conninfo(), clientinfo := emqx_types:clientinfo() @@ -133,7 +124,7 @@ handle_call( ) -> ok = emqx_session:takeover(Session), %% TODO: Should not drain deliver here (side effect) - Delivers = emqx_misc:drain_deliver(), + Delivers = emqx_utils:drain_deliver(), AllPendings = lists:append(Delivers, Pendings), ?tp( debug, @@ -156,7 +147,7 @@ handle_call(Req, _From, Channel) -> {reply, ignored, Channel}. handle_info(Deliver = {deliver, _Topic, _Msg}, Channel) -> - Delivers = [Deliver | emqx_misc:drain_deliver()], + Delivers = [Deliver | emqx_utils:drain_deliver()], {noreply, handle_deliver(Delivers, Channel)}; handle_info(expire_session, Channel) -> {stop, expired, Channel}; @@ -186,7 +177,6 @@ code_change(_OldVsn, Channel, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- -%% TODO: sync with emqx_channel handle_deliver( Delivers, #{ @@ -239,7 +229,7 @@ set_expiry_timer(#{conninfo := ConnInfo} = Channel) -> open_session(ConnInfo, #{clientid := ClientId} = ClientInfo) -> Channel = channel(ConnInfo, ClientInfo), - case emqx_cm:open_session(false, ClientInfo, ConnInfo) of + case emqx_cm:open_session(_CleanSession = false, ClientInfo, ConnInfo) of {ok, #{present := false}} -> ?SLOG( info, @@ -259,7 +249,7 @@ open_session(ConnInfo, #{clientid := ClientId} = ClientInfo) -> node => node() } ), - Pendings1 = lists:usort(lists:append(Pendings0, emqx_misc:drain_deliver())), + Pendings1 = lists:usort(lists:append(Pendings0, emqx_utils:drain_deliver())), NSession = emqx_session:enqueue( ClientInfo, emqx_session:ignore_local( @@ -352,7 +342,7 @@ info(Channel) -> #{ conninfo => maps:get(conninfo, Channel, undefined), clientinfo => maps:get(clientinfo, Channel, undefined), - session => maybe_apply( + session => emqx_utils:maybe_apply( fun emqx_session:info/1, maps:get(session, Channel, undefined) ), diff --git a/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl b/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl index 0574ccec3..22b694d77 100644 --- a/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl +++ b/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl @@ -362,13 +362,77 @@ t_will_msg(_Config) -> ok = emqtt:disconnect(C). +t_ws_conn(_Config) -> + erlang:process_flag(trap_exit, true), + + ClientId = <<"ws_client">>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {clientid, ClientId}, + {port, 8083}, + {ws_path, "/mqtt"} + ]), + {ok, _} = emqtt:ws_connect(C), + + ok = emqx_eviction_agent:enable(test_eviction, undefined), + + ?assertEqual( + 1, + emqx_eviction_agent:connection_count() + ), + + ?assertWaitEvent( + ok = emqx_eviction_agent:evict_connections(1), + #{?snk_kind := emqx_cm_connected_client_count_dec}, + 1000 + ), + + ?assertEqual( + 0, + emqx_eviction_agent:connection_count() + ). + +-ifndef(BUILD_WITHOUT_QUIC). + +t_quic_conn(_Config) -> + erlang:process_flag(trap_exit, true), + + QuicPort = emqx_common_test_helpers:select_free_port(quic), + application:ensure_all_started(quicer), + emqx_common_test_helpers:ensure_quic_listener(?MODULE, QuicPort), + + ClientId = <<"quic_client">>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {clientid, ClientId}, + {port, QuicPort} + ]), + {ok, _} = emqtt:quic_connect(C), + + ok = emqx_eviction_agent:enable(test_eviction, undefined), + + ?assertEqual( + 1, + emqx_eviction_agent:connection_count() + ), + + ?assertWaitEvent( + ok = emqx_eviction_agent:evict_connections(1), + #{?snk_kind := emqx_cm_connected_client_count_dec}, + 1000 + ), + + ?assertEqual( + 0, + emqx_eviction_agent:connection_count() + ). + +-endif. + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- -% sn_connect_and_subscribe(ClientId, Topic) -> -% emqx_eviction_agent_test_helpers:sn_connect_and_subscribe(ClientId, Topic). - assert_receive_publish([]) -> ok; assert_receive_publish([#{payload := Msg, topic := Topic} | Rest]) -> diff --git a/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl b/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl index 8f88ebf97..3953ec3e2 100644 --- a/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl +++ b/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl @@ -81,12 +81,11 @@ start_cluster(NamesWithPorts, Apps, Env) -> NamesWithPorts ), Opts0 = [ - {env, [{emqx, boot_modules, [broker, listeners]}]}, + {env, [{emqx, boot_modules, [broker, listeners]}] ++ Env}, {apps, Apps}, {conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]] ++ - [{[rpc, mode], async}]}, - {env, Env} + [{[rpc, mode], async}]} ], Cluster = emqx_common_test_helpers:emqx_cluster( Specs, @@ -99,12 +98,6 @@ start_cluster(NamesWithPorts, Apps, Env) -> } || {Name, Opts} <- Cluster ], - ok = lists:foreach( - fun({Node, _Port}) -> - snabbkaffe:forward_trace(Node) - end, - NodesWithPorts - ), NodesWithPorts. stop_cluster(NodesWithPorts, Apps) -> diff --git a/lib-ee/emqx_node_rebalance/i18n/emqx_node_rebalance_api_i18n.conf b/lib-ee/emqx_node_rebalance/i18n/emqx_node_rebalance_api_i18n.conf deleted file mode 100644 index f5f161a92..000000000 --- a/lib-ee/emqx_node_rebalance/i18n/emqx_node_rebalance_api_i18n.conf +++ /dev/null @@ -1,490 +0,0 @@ -emqx_node_rebalance_api { - - ## API Request Fields - - load_rebalance_status { - desc { - en: "Get rebalance status of the current node" - zh: "获取当前节点的rebalance状态" - } - label { - en: "Get rebalance status" - zh: "获取rebalance状态" - } - } - - load_rebalance_global_status { - desc { - en: "Get status of all rebalance/evacuation processes across the cluster" - zh: "获取集群中所有rebalance/evacuation进程的状态" - } - label { - en: "Get global rebalance status" - zh: "获取全局rebalance状态" - } - } - - load_rebalance_availability_check { - desc { - en: "Check if the node is being evacuated or rebalanced" - zh: "检查节点是否正在被evacuate或rebalance" - } - label { - en: "Availability check" - zh: "可用性检查" - } - } - - load_rebalance_start { - desc { - en: "Start rebalance process" - zh: "启动rebalance进程" - } - label { - en: "Start rebalance" - zh: "启动rebalance" - } - } - - load_rebalance_stop { - desc { - en: "Stop rebalance process" - zh: "停止rebalance进程" - } - label { - en: "Stop rebalance" - zh: "停止rebalance" - } - } - - load_rebalance_evacuation_start { - desc { - en: "Start evacuation process" - zh: "启动evacuation进程" - } - label { - en: "Start evacuation" - zh: "启动evacuation" - } - } - - load_rebalance_evacuation_stop { - desc { - en: "Stop evacuation process" - zh: "停止evacuation进程" - } - label { - en: "Stop evacuation" - zh: "停止evacuation" - } - } - - param_node { - desc { - en: "Node name" - zh: "节点名称" - } - label { - en: "Node name" - zh: "节点名称" - } - } - - wait_health_check { - desc { - en: "Time to wait before starting the rebalance process, in seconds" - zh: "启动rebalance进程前等待的时间，单位为秒" - } - label { - en: "Wait health check" - zh: "等待健康检查" - } - } - - conn_evict_rate { - desc { - en: "The rate of evicting connections, in connections per second" - zh: "逐出连接的速率，以每秒连接数表示" - } - label { - en: "Connection eviction rate" - zh: "连接驱逐率" - } - } - - sess_evict_rate { - desc { - en: "The rate of evicting sessions, in sessions per second" - zh: "逐出会话的速率，以每秒会话为单位" - } - label { - en: "Session eviction rate" - zh: "会话驱逐率" - } - } - - abs_conn_threshold { - desc { - en: "Maximum desired difference between the number of connections on the node and the average number of connections on the recipient nodes" - zh: "节点上的连接数与接收节点上的平均连接数之间的最大期望差值" - } - label { - en: "Absolute connection threshold" - zh: "绝对连接阈值" - } - } - - rel_conn_threshold { - desc { - en: "Maximum desired fraction between the number of connections on the node and the average number of connections on the recipient nodes" - zh: "节点上的连接数与接收节点上的平均连接数之间的最大期望分数" - } - label { - en: "Relative connection threshold" - zh: "相对连接阈值" - } - } - - abs_sess_threshold { - desc { - en: "Maximum desired difference between the number of sessions on the node and the average number of sessions on the recipient nodes" - zh: "节点上的会话数与接收节点上的平均会话数之间的最大期望差异" - } - label { - en: "Absolute session threshold" - zh: "绝对会话阈值" - } - } - - rel_sess_threshold { - desc { - en: "Maximum desired fraction between the number of sessions on the node and the average number of sessions on the recipient nodes" - zh: "节点上的会话数与接收节点上的平均会话数之间的最大期望分数" - } - label { - en: "Relative session threshold" - zh: "相对会话阈值" - } - } - - wait_takeover { - desc { - en: "Time to wait before starting session evacuation process, in seconds" - zh: "开始会话疏散过程之前等待的时间，以秒为单位" - } - label { - en: "Wait takeover" - zh: "等待接管" - } - } - - redirect_to { - desc { - en: "Server reference to redirect clients to (MQTTv5 Server redirection)" - zh: "将客户端重定向到的服务器参考（MQTTv5 服务器重定向）" - } - label { - en: "Redirect to" - zh: "重定向至" - } - } - - migrate_to { - desc { - en: "Nodes to migrate sessions to" - zh: "将会话迁移到的节点" - } - label { - en: "Migrate to" - zh: "迁移到" - } - } - - rebalance_nodes { - desc { - en: "Nodes to participate in rebalance" - zh: "参与rebalance的节点" - } - label { - en: "Rebalance nodes" - zh: "重新平衡节点" - } - } - - ## API Response Fields - - local_status_enabled { - desc { - en: "Whether the node is being evacuated" - zh: "节点是否正在撤离" - } - label { - en: "Local evacuation status" - zh: "当地避难状况" - } - } - - local_status_process { - desc { - en: "The process that is being performed on the node: evacuation or rebalance" - zh: "正在节点上执行的过程：疏散或重新平衡" - } - label { - en: "Node process" - zh: "节点进程" - } - } - - local_status_state { - desc { - en: "The state of the process that is being performed on the node" - zh: "正在节点上执行的进程的状态" - } - label { - en: "Rebalance/evacuation current state" - zh: "重新平衡/疏散当前状态" - } - } - - local_status_coordinator_node { - desc { - en: "The node that is coordinating rebalance process" - zh: "协调再平衡过程的节点" - } - label { - en: "Coordinator node" - zh: "协调节点" - } - } - - local_status_connection_eviction_rate { - desc { - en: "The rate of evicting connections, in connections per second" - zh: "逐出连接的速率，以每秒连接数表示" - } - label { - en: "Connection eviction rate" - zh: "连接驱逐率" - } - } - - local_status_session_eviction_rate { - desc { - en: "The rate of evicting sessions, in sessions per second" - zh: "逐出会话的速率，以每秒会话为单位" - } - label { - en: "Session eviction rate" - zh: "会话驱逐率" - } - } - - local_status_connection_goal { - desc { - en: "The number of connections that the node should have after the rebalance/evacuation process" - zh: "节点在重新平衡/疏散过程后应该拥有的连接数" - } - label { - en: "Connection goal" - zh: "连接目标" - } - } - - local_status_session_goal { - desc { - en: "The number of sessions that the node should have after the evacuation process" - zh: "疏散过程后节点应有的会话数" - } - label { - en: "Session goal" - zh: "会话目标" - } - } - - local_status_disconnected_session_goal { - desc { - en: "The number of disconnected sessions that the node should have after the rebalance process" - zh: "重新平衡过程后节点应具有的断开连接的会话数" - } - label { - en: "Disconnected session goal" - zh: "断开连接的会话目标" - } - } - - local_status_session_recipients { - desc { - en: "List of nodes to which sessions are being evacuated" - zh: "会话被疏散到的节点列表" - } - label { - en: "Session recipients" - zh: "会话收件人" - } - } - - local_status_recipients { - desc { - en: "List of nodes to which connections/sessions are being evacuated during rebalance" - zh: "在重新平衡期间连接/会话被疏散到的节点列表" - } - label { - en: "Recipients" - zh: "收件人" - } - } - - local_status_stats { - desc { - en: "Statistics of the evacuation/rebalance process" - zh: "疏散/再平衡过程的统计" - } - label { - en: "Statistics" - zh: "统计数据" - } - } - - status_stats_initial_connected { - desc { - en: "The number of connections on the node before the evacuation/rebalance process" - zh: "疏散/重新平衡过程之前节点上的连接数" - } - label { - en: "Initial connected" - zh: "初始连接" - } - } - - status_stats_current_connected { - desc { - en: "Current number of connections on the node" - zh: "节点上的当前连接数" - } - label { - en: "Current connections" - zh: "当前连接" - } - } - - status_stats_initial_sessions { - desc { - en: "The number of sessions on the node before the evacuation/rebalance process" - zh: "疏散/重新平衡过程之前节点上的会话数" - } - label { - en: "Initial sessions" - zh: "初始会话" - } - } - - status_stats_current_sessions { - desc { - en: "Current number of sessions on the node" - zh: "节点上的当前会话数" - } - label { - en: "Current sessions" - zh: "当前会话" - } - } - - status_stats_current_disconnected_sessions { - desc { - en: "Current number of disconnected sessions on the node" - zh: "节点上当前断开连接的会话数" - } - label { - en: "Current disconnected sessions" - zh: "当前断开连接的会话" - } - } - - coordinator_status_donors { - desc { - en: "List of nodes from which connections/sessions are being evacuated" - zh: "正在疏散连接/会话的节点列表" - } - label { - en: "Donors" - zh: "捐助者" - } - } - - coordinator_status_donor_conn_avg { - desc { - en: "Average number of connections per donor node" - zh: "每个供体节点的平均连接数" - } - label { - en: "Donor connections average" - zh: "捐助者连接平均值" - } - } - - coordinator_status_donor_sess_avg { - desc { - en: "Average number of sessions per donor node" - zh: "每个供体节点的平均会话数" - } - label { - en: "Donor sessions average" - zh: "平均捐助会议" - } - } - - coordinator_status_node { - desc { - en: "The node that is coordinating the evacuation/rebalance process" - zh: "协调疏散/再平衡过程的节点" - } - label { - en: "Coordinator node" - zh: "协调节点" - } - } - - evacuation_status_node { - desc { - en: "The node that is being evacuated" - zh: "正在撤离的节点" - } - label { - en: "Evacuated node" - zh: "疏散节点" - } - } - - global_status_evacuations { - desc { - en: "List of nodes that are being evacuated" - zh: "正在撤离的节点列表" - } - label { - en: "Evacuations" - zh: "疏散" - } - } - - global_status_rebalances { - desc { - en: "List of nodes that coordinate a rebalance" - zh: "协调再平衡的节点列表" - } - label { - en: "Rebalances" - zh: "再平衡" - } - } - - empty_response { - desc { - en: "The response is empty" - zh: "响应为空" - } - label { - en: "Empty response" - zh: "空响应" - } - } -} diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_api.erl b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_api.erl index fa322d146..1f6328a63 100644 --- a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_api.erl +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_api.erl @@ -8,6 +8,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_utils/include/emqx_utils_api.hrl"). %% Swagger specs from hocon schema -export([ @@ -44,9 +45,9 @@ -import(emqx_dashboard_swagger, [error_codes/2]). -define(BAD_REQUEST, 'BAD_REQUEST'). --define(NODE_UNAVAILABLE, 'NODE_UNAVAILABLE'). -define(NODE_EVACUATING, 'NODE_EVACUATING'). -define(RPC_ERROR, 'RPC_ERROR'). +-define(NOT_FOUND, 'NOT_FOUND'). %%-------------------------------------------------------------------- %% API Spec @@ -120,7 +121,8 @@ schema("/load_rebalance/:node/start") -> ), responses => #{ 200 => response_schema(), - 400 => error_codes([?BAD_REQUEST, ?NODE_UNAVAILABLE], <<"Bad Request">>) + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) } } }; @@ -134,7 +136,8 @@ schema("/load_rebalance/:node/stop") -> parameters => [param_node()], responses => #{ 200 => response_schema(), - 400 => error_codes([?BAD_REQUEST, ?NODE_UNAVAILABLE], <<"Bad Request">>) + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) } } }; @@ -153,7 +156,8 @@ schema("/load_rebalance/:node/evacuation/start") -> ), responses => #{ 200 => response_schema(), - 400 => error_codes([?BAD_REQUEST, ?NODE_UNAVAILABLE], <<"Bad Request">>) + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) } } }; @@ -167,7 +171,8 @@ schema("/load_rebalance/:node/evacuation/stop") -> parameters => [param_node()], responses => #{ 200 => response_schema(), - 400 => error_codes([?BAD_REQUEST, ?NODE_UNAVAILABLE], <<"Bad Request">>) + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) } } }. @@ -205,7 +210,7 @@ schema("/load_rebalance/:node/evacuation/stop") -> end. '/load_rebalance/:node/start'(post, #{bindings := #{node := NodeBin}, body := Params0}) -> - with_node(NodeBin, fun(Node) -> + emqx_utils_api:with_node(NodeBin, fun(Node) -> Params1 = translate(rebalance_start, Params0), with_nodes_at_key(nodes, Params1, fun(Params2) -> wrap_rpc( @@ -215,7 +220,7 @@ schema("/load_rebalance/:node/evacuation/stop") -> end). '/load_rebalance/:node/stop'(post, #{bindings := #{node := NodeBin}}) -> - with_node(NodeBin, fun(Node) -> + emqx_utils_api:with_node(NodeBin, fun(Node) -> wrap_rpc( Node, emqx_node_rebalance_api_proto_v1:node_rebalance_stop(Node) ) @@ -224,7 +229,7 @@ schema("/load_rebalance/:node/evacuation/stop") -> '/load_rebalance/:node/evacuation/start'(post, #{ bindings := #{node := NodeBin}, body := Params0 }) -> - with_node(NodeBin, fun(Node) -> + emqx_utils_api:with_node(NodeBin, fun(Node) -> Params1 = translate(rebalance_evacuation_start, Params0), with_nodes_at_key(migrate_to, Params1, fun(Params2) -> wrap_rpc( @@ -237,7 +242,7 @@ schema("/load_rebalance/:node/evacuation/stop") -> end). '/load_rebalance/:node/evacuation/stop'(post, #{bindings := #{node := NodeBin}}) -> - with_node(NodeBin, fun(Node) -> + emqx_utils_api:with_node(NodeBin, fun(Node) -> wrap_rpc( Node, emqx_node_rebalance_api_proto_v1:node_rebalance_evacuation_stop(Node) ) @@ -288,19 +293,13 @@ validate_nodes(Key, Params) when is_map_key(Key, Params) -> validate_nodes(_Key, Params) -> {ok, Params}. -with_node(BinNode, Fun) -> - case parse_node(BinNode) of - {ok, Node} -> Fun(Node); - {error, _} -> error_response(400, ?BAD_REQUEST, [<<"Invalid node: ">>, BinNode]) - end. - with_nodes_at_key(Key, Params, Fun) -> Res = validate_nodes(Key, Params), case Res of {ok, Params1} -> Fun(Params1); {error, {unavailable, Nodes}} -> - error_response(400, ?NODE_UNAVAILABLE, io_lib:format("Nodes unavailable: ~p", [Nodes])); + error_response(400, ?NOT_FOUND, io_lib:format("Nodes unavailable: ~p", [Nodes])); {error, {invalid, Nodes}} -> error_response(400, ?BAD_REQUEST, io_lib:format("Invalid nodes: ~p", [Nodes])) end. @@ -322,10 +321,7 @@ format_as_map_list(List) -> ). error_response(HttpCode, Code, Message) -> - {HttpCode, #{ - code => atom_to_binary(Code), - message => iolist_to_binary(Message) - }}. + {HttpCode, ?ERROR_MSG(Code, Message)}. without(Keys, Props) -> lists:filter( @@ -470,11 +466,10 @@ fields(rebalance_evacuation_start) -> )}, {"migrate_to", mk( - list(binary()), + nonempty_list(binary()), #{ desc => ?DESC(migrate_to), - required => false, - validator => [fun(Values) -> length(Values) > 0 end] + required => false } )} ]; diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl index a2706f13b..3bafb9ffe 100644 --- a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl @@ -43,7 +43,7 @@ cli(["start" | StartArgs]) -> false end; cli(["node-status", NodeStr]) -> - case emqx_misc:safe_to_existing_atom(NodeStr, utf8) of + case emqx_utils:safe_to_existing_atom(NodeStr, utf8) of {ok, Node} -> node_status(emqx_node_rebalance_status:local_status(Node)); {error, _} -> @@ -297,7 +297,7 @@ strings_to_atoms(Strings) -> strings_to_atoms([], Atoms, Invalid) -> {lists:reverse(Atoms), lists:reverse(Invalid)}; strings_to_atoms([Str | Rest], Atoms, Invalid) -> - case emqx_misc:safe_to_existing_atom(Str, utf8) of + case emqx_utils:safe_to_existing_atom(Str, utf8) of {ok, Atom} -> strings_to_atoms(Rest, [Atom | Atoms], Invalid); {error, _} -> diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl index 3fc9faeea..6b145c699 100644 --- a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl @@ -55,7 +55,7 @@ save( Filepath = evacuation_filepath(), case filelib:ensure_dir(Filepath) of ok -> - JsonData = emqx_json:encode( + JsonData = emqx_utils_json:encode( prepare_for_encode(maps:with(persist_keys(), Data)), [pretty] ), @@ -72,7 +72,7 @@ clear() -> read(DefaultOpts) -> case file:read_file(evacuation_filepath()) of {ok, Data} -> - case emqx_json:safe_decode(Data, [return_maps]) of + case emqx_utils_json:safe_decode(Data, [return_maps]) of {ok, Map} when is_map(Map) -> {ok, map_to_opts(DefaultOpts, Map)}; _NotAMap -> diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_status.erl b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_status.erl index 63675a3da..1d45d64e8 100644 --- a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_status.erl +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_status.erl @@ -208,7 +208,7 @@ format_local_status_field({session_goal, SessGoal}) -> format_local_status_field({disconnected_session_goal, DisconnSessGoal}) -> io_lib:format("Disconnected session goal: ~p~n", [DisconnSessGoal]); format_local_status_field({session_recipients, SessionRecipients}) -> - io_lib:format("Session recipient nodes: ~p~n", [SessionRecipients]); + io_lib:format("Session recipient nodes: ~p~n", [SessionRecipients]); format_local_status_field({recipients, Recipients}) -> io_lib:format("Recipient nodes: ~p~n", [Recipients]); format_local_status_field({donors, Donors}) -> diff --git a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl index 21608b8bc..d8202a33e 100644 --- a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl +++ b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl @@ -88,7 +88,7 @@ t_start_evacuation_validation(Config) -> BadOpts ), ?assertMatch( - {ok, 400, #{}}, + {ok, 404, #{}}, api_post( ["load_rebalance", "bad@node", "evacuation", "start"], #{} @@ -148,7 +148,7 @@ t_start_rebalance_validation(Config) -> BadOpts ), ?assertMatch( - {ok, 400, #{}}, + {ok, 404, #{}}, api_post( ["load_rebalance", "bad@node", "start"], #{} @@ -346,7 +346,7 @@ t_start_stop_rebalance(Config) -> } ] }}, - api_get(["load_rebalance", "global_status"]) + GlobalStatusResponse ), ?assertMatch( diff --git a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl index cdafad97a..5d774ba7c 100644 --- a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl +++ b/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl @@ -22,21 +22,20 @@ all() -> [{group, one_node}, {group, two_node}]. groups() -> [ - {one_node, [], [ - t_agent_busy, - t_already_started, - t_not_started, - t_start, - t_persistence, - t_unknown_messages - ]}, - {two_node, [], [ - t_conn_evicted, - t_migrate_to, - t_session_evicted - ]} + {one_node, [], one_node_cases()}, + {two_node, [], two_node_cases()} ]. +two_node_cases() -> + [ + t_conn_evicted, + t_migrate_to, + t_session_evicted + ]. + +one_node_cases() -> + emqx_common_test_helpers:all(?MODULE) -- two_node_cases(). + init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps([]), Config. diff --git a/rel/i18n/emqx_eviction_agent_api.hocon b/rel/i18n/emqx_eviction_agent_api.hocon new file mode 100644 index 000000000..40566fca6 --- /dev/null +++ b/rel/i18n/emqx_eviction_agent_api.hocon @@ -0,0 +1,9 @@ +emqx_eviction_agent_api { + +node_eviction_status_get.desc: +"""Get the node eviction status""" + +node_eviction_status_get.label: +"""Node Eviction Status""" + +} diff --git a/rel/i18n/emqx_node_rebalance_api.hocon b/rel/i18n/emqx_node_rebalance_api.hocon new file mode 100644 index 000000000..51d0fa8bc --- /dev/null +++ b/rel/i18n/emqx_node_rebalance_api.hocon @@ -0,0 +1,267 @@ +emqx_node_rebalance_api { + +load_rebalance_status.desc: +"""Get rebalance status of the current node""" + +load_rebalance_status.label: +"""Get rebalance status""" + +load_rebalance_global_status.desc: +"""Get status of all rebalance/evacuation processes across the cluster""" + +load_rebalance_global_status.label: +"""Get global rebalance status""" + +load_rebalance_availability_check.desc: +"""Check if the node is being evacuated or rebalanced""" + +load_rebalance_availability_check.label: +"""Availability check""" + +load_rebalance_start.desc: +"""Start rebalance process""" + +load_rebalance_start.label: +"""Start rebalance""" + +load_rebalance_stop.desc: +"""Stop rebalance process""" + +load_rebalance_stop.label: +"""Stop rebalance""" + +load_rebalance_evacuation_start.desc: +"""Start evacuation process""" + +load_rebalance_evacuation_start.label: +"""Start evacuation""" + +load_rebalance_evacuation_stop.desc: +"""Stop evacuation process""" + +load_rebalance_evacuation_stop.label: +"""Stop evacuation""" + +param_node.desc: +"""Node name""" + +param_node.label: +"""Node name""" + +wait_health_check.desc: +"""Time to wait before starting the rebalance process, in seconds""" + +wait_health_check.label: +"""Wait health check""" + +conn_evict_rate.desc: +"""The rate of evicting connections, in connections per second""" + +conn_evict_rate.label: +"""Connection eviction rate""" + +sess_evict_rate.desc: +"""The rate of evicting sessions, in sessions per second""" + +sess_evict_rate.label: +"""Session eviction rate""" + +abs_conn_threshold.desc: +"""Maximum desired difference between the number of connections on the node and the average number of connections on the recipient nodes""" + +abs_conn_threshold.label: +"""Absolute connection threshold""" + +rel_conn_threshold.desc: +"""Maximum desired fraction between the number of connections on the node and the average number of connections on the recipient nodes""" + +rel_conn_threshold.label: +"""Relative connection threshold""" + +abs_sess_threshold.desc: +"""Maximum desired difference between the number of sessions on the node and the average number of sessions on the recipient nodes""" + +abs_sess_threshold.label: +"""Absolute session threshold""" + +rel_sess_threshold.desc: +"""Maximum desired fraction between the number of sessions on the node and the average number of sessions on the recipient nodes""" + +rel_sess_threshold.label: +"""Relative session threshold""" + +wait_takeover.desc: +"""Time to wait before starting session evacuation process, in seconds""" + +wait_takeover.label: +"""Wait takeover""" + +redirect_to.desc: +"""Server reference to redirect clients to (MQTTv5 Server redirection)""" + +redirect_to.label: +"""Redirect to""" + +migrate_to.desc: +"""Nodes to migrate sessions to""" + +migrate_to.label: +"""Migrate to""" + +rebalance_nodes.desc: +"""Nodes to participate in rebalance""" + +rebalance_nodes.label: +"""Rebalance nodes""" + +local_status_enabled.desc: +"""Whether the node is being evacuated""" + +local_status_enabled.label: +"""Local evacuation status""" + +local_status_process.desc: +"""The process that is being performed on the node: evacuation or rebalance""" + +local_status_process.label: +"""Node process""" + +local_status_state.desc: +"""The state of the process that is being performed on the node""" + +local_status_state.label: +"""Rebalance/evacuation current state""" + +local_status_coordinator_node.desc: +"""The node that is coordinating rebalance process""" + +local_status_coordinator_node.label: +"""Coordinator node""" + +local_status_connection_eviction_rate.desc: +"""The rate of evicting connections, in connections per second""" + +local_status_connection_eviction_rate.label: +"""Connection eviction rate""" + +local_status_session_eviction_rate.desc: +"""The rate of evicting sessions, in sessions per second""" + +local_status_session_eviction_rate.label: +"""Session eviction rate""" + +local_status_connection_goal.desc: +"""The number of connections that the node should have after the rebalance/evacuation process""" + +local_status_connection_goal.label: +"""Connection goal""" + +local_status_session_goal.desc: +"""The number of sessions that the node should have after the evacuation process""" + +local_status_session_goal.label: +"""Session goal""" + +local_status_disconnected_session_goal.desc: +"""The number of disconnected sessions that the node should have after the rebalance process""" + +local_status_disconnected_session_goal.label: +"""Disconnected session goal""" + +local_status_session_recipients.desc: +"""List of nodes to which sessions are being evacuated""" + +local_status_session_recipients.label: +"""Session recipients""" + +local_status_recipients.desc: +"""List of nodes to which connections/sessions are being evacuated during rebalance""" + +local_status_recipients.label: +"""Recipients""" + +local_status_stats.desc: +"""Statistics of the evacuation/rebalance process""" + +local_status_stats.label: +"""Statistics""" + +status_stats_initial_connected.desc: +"""The number of connections on the node before the evacuation/rebalance process""" + +status_stats_initial_connected.label: +"""Initial connected""" + +status_stats_current_connected.desc: +"""Current number of connections on the node""" + +status_stats_current_connected.label: +"""Current connections""" + +status_stats_initial_sessions.desc: +"""The number of sessions on the node before the evacuation/rebalance process""" + +status_stats_initial_sessions.label: +"""Initial sessions""" + +status_stats_current_sessions.desc: +"""Current number of sessions on the node""" + +status_stats_current_sessions.label: +"""Current sessions""" + +status_stats_current_disconnected_sessions.desc: +"""Current number of disconnected sessions on the node""" + +status_stats_current_disconnected_sessions.label: +"""Current disconnected sessions""" + +coordinator_status_donors.desc: +"""List of nodes from which connections/sessions are being evacuated""" + +coordinator_status_donors.label: +"""Donors""" + +coordinator_status_donor_conn_avg.desc: +"""Average number of connections per donor node""" + +coordinator_status_donor_conn_avg.label: +"""Donor connections average""" + +coordinator_status_donor_sess_avg.desc: +"""Average number of sessions per donor node""" + +coordinator_status_donor_sess_avg.label: +"""Donor sessions average""" + +coordinator_status_node.desc: +"""The node that is coordinating the evacuation/rebalance process""" + +coordinator_status_node.label: +"""Coordinator node""" + +evacuation_status_node.desc: +"""The node that is being evacuated""" + +evacuation_status_node.label: +"""Evacuated node""" + +global_status_evacuations.desc: +"""List of nodes that are being evacuated""" + +global_status_evacuations.label: +"""Evacuations""" + +global_status_rebalances.desc: +"""List of nodes that coordinate a rebalance""" + +global_status_rebalances.label: +"""Rebalances""" + +empty_response.desc: +"""The response is empty""" + +empty_response.label: +"""Empty response""" + +} diff --git a/rel/i18n/zh/emqx_eviction_agent_api.hocon b/rel/i18n/zh/emqx_eviction_agent_api.hocon new file mode 100644 index 000000000..a4d9f5c12 --- /dev/null +++ b/rel/i18n/zh/emqx_eviction_agent_api.hocon @@ -0,0 +1,9 @@ +emqx_eviction_agent_api { + +node_eviction_status_get.desc: +"""获取节点驱逐状态""" + +node_eviction_status_get.label: +"""节点驱逐状态""" + +} diff --git a/rel/i18n/zh/emqx_node_rebalance_api.hocon b/rel/i18n/zh/emqx_node_rebalance_api.hocon new file mode 100644 index 000000000..3066158b3 --- /dev/null +++ b/rel/i18n/zh/emqx_node_rebalance_api.hocon @@ -0,0 +1,267 @@ +emqx_node_rebalance_api { + +load_rebalance_status.desc: +"""获取当前节点的rebalance状态""" + +load_rebalance_status.label: +"""获取rebalance状态""" + +load_rebalance_global_status.desc: +"""获取集群中所有rebalance/evacuation进程的状态""" + +load_rebalance_global_status.label: +"""获取全局rebalance状态""" + +load_rebalance_availability_check.desc: +"""检查节点是否正在被evacuate或rebalance""" + +load_rebalance_availability_check.label: +"""可用性检查""" + +load_rebalance_start.desc: +"""启动rebalance进程""" + +load_rebalance_start.label: +"""启动rebalance""" + +load_rebalance_stop.desc: +"""停止rebalance进程""" + +load_rebalance_stop.label: +"""停止rebalance""" + +load_rebalance_evacuation_start.desc: +"""启动evacuation进程""" + +load_rebalance_evacuation_start.label: +"""启动evacuation""" + +load_rebalance_evacuation_stop.desc: +"""停止evacuation进程""" + +load_rebalance_evacuation_stop.label: +"""停止evacuation""" + +param_node.desc: +"""节点名称""" + +param_node.label: +"""节点名称""" + +wait_health_check.desc: +"""启动rebalance进程前等待的时间，单位为秒""" + +wait_health_check.label: +"""等待健康检查""" + +conn_evict_rate.desc: +"""逐出连接的速率，以每秒连接数表示""" + +conn_evict_rate.label: +"""连接驱逐率""" + +sess_evict_rate.desc: +"""逐出会话的速率，以每秒会话为单位""" + +sess_evict_rate.label: +"""会话驱逐率""" + +abs_conn_threshold.desc: +"""节点上的连接数与接收节点上的平均连接数之间的最大期望差值""" + +abs_conn_threshold.label: +"""绝对连接阈值""" + +rel_conn_threshold.desc: +"""节点上的连接数与接收节点上的平均连接数之间的最大期望分数""" + +rel_conn_threshold.label: +"""相对连接阈值""" + +abs_sess_threshold.desc: +"""节点上的会话数与接收节点上的平均会话数之间的最大期望差异""" + +abs_sess_threshold.label: +"""绝对会话阈值""" + +rel_sess_threshold.desc: +"""节点上的会话数与接收节点上的平均会话数之间的最大期望分数""" + +rel_sess_threshold.label: +"""相对会话阈值""" + +wait_takeover.desc: +"""开始会话疏散过程之前等待的时间，以秒为单位""" + +wait_takeover.label: +"""等待接管""" + +redirect_to.desc: +"""将客户端重定向到的服务器参考（MQTTv5 服务器重定向）""" + +redirect_to.label: +"""重定向至""" + +migrate_to.desc: +"""将会话迁移到的节点""" + +migrate_to.label: +"""迁移到""" + +rebalance_nodes.desc: +"""参与rebalance的节点""" + +rebalance_nodes.label: +"""重新平衡节点""" + +local_status_enabled.desc: +"""节点是否正在撤离""" + +local_status_enabled.label: +"""当地避难状况""" + +local_status_process.desc: +"""正在节点上执行的过程：疏散或重新平衡""" + +local_status_process.label: +"""节点进程""" + +local_status_state.desc: +"""正在节点上执行的进程的状态""" + +local_status_state.label: +"""重新平衡/疏散当前状态""" + +local_status_coordinator_node.desc: +"""协调再平衡过程的节点""" + +local_status_coordinator_node.label: +"""协调节点""" + +local_status_connection_eviction_rate.desc: +"""逐出连接的速率，以每秒连接数表示""" + +local_status_connection_eviction_rate.label: +"""连接驱逐率""" + +local_status_session_eviction_rate.desc: +"""逐出会话的速率，以每秒会话为单位""" + +local_status_session_eviction_rate.label: +"""会话驱逐率""" + +local_status_connection_goal.desc: +"""节点在重新平衡/疏散过程后应该拥有的连接数""" + +local_status_connection_goal.label: +"""连接目标""" + +local_status_session_goal.desc: +"""疏散过程后节点应有的会话数""" + +local_status_session_goal.label: +"""会话目标""" + +local_status_disconnected_session_goal.desc: +"""重新平衡过程后节点应具有的断开连接的会话数""" + +local_status_disconnected_session_goal.label: +"""断开连接的会话目标""" + +local_status_session_recipients.desc: +"""会话被疏散到的节点列表""" + +local_status_session_recipients.label: +"""会话收件人""" + +local_status_recipients.desc: +"""在重新平衡期间连接/会话被疏散到的节点列表""" + +local_status_recipients.label: +"""收件人""" + +local_status_stats.desc: +"""疏散/再平衡过程的统计""" + +local_status_stats.label: +"""统计数据""" + +status_stats_initial_connected.desc: +"""疏散/重新平衡过程之前节点上的连接数""" + +status_stats_initial_connected.label: +"""初始连接""" + +status_stats_current_connected.desc: +"""节点上的当前连接数""" + +status_stats_current_connected.label: +"""当前连接""" + +status_stats_initial_sessions.desc: +"""疏散/重新平衡过程之前节点上的会话数""" + +status_stats_initial_sessions.label: +"""初始会话""" + +status_stats_current_sessions.desc: +"""节点上的当前会话数""" + +status_stats_current_sessions.label: +"""当前会话""" + +status_stats_current_disconnected_sessions.desc: +"""节点上当前断开连接的会话数""" + +status_stats_current_disconnected_sessions.label: +"""当前断开连接的会话""" + +coordinator_status_donors.desc: +"""正在疏散连接/会话的节点列表""" + +coordinator_status_donors.label: +"""捐助者""" + +coordinator_status_donor_conn_avg.desc: +"""每个供体节点的平均连接数""" + +coordinator_status_donor_conn_avg.label: +"""捐助者连接平均值""" + +coordinator_status_donor_sess_avg.desc: +"""每个供体节点的平均会话数""" + +coordinator_status_donor_sess_avg.label: +"""平均捐助会议""" + +coordinator_status_node.desc: +"""协调疏散/再平衡过程的节点""" + +coordinator_status_node.label: +"""协调节点""" + +evacuation_status_node.desc: +"""正在撤离的节点""" + +evacuation_status_node.label: +"""疏散节点""" + +global_status_evacuations.desc: +"""正在撤离的节点列表""" + +global_status_evacuations.label: +"""疏散""" + +global_status_rebalances.desc: +"""协调再平衡的节点列表""" + +global_status_rebalances.label: +"""再平衡""" + +empty_response.desc: +"""响应为空""" + +empty_response.label: +"""空响应""" +} + From ffed8a132bb806a142608db2c40ff5526609bf87 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 May 2023 14:13:49 +0500 Subject: [PATCH 042/144] chore(rebalance): review fixes Co-authored-by: Zaiming (Stone) Shi --- lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.app.src | 3 +-- rel/i18n/emqx_node_rebalance_api.hocon | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.app.src b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.app.src index 9673e4fda..381001b87 100644 --- a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.app.src +++ b/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.app.src @@ -14,9 +14,8 @@ {mod, {emqx_node_rebalance_app, []}}, {env, []}, {modules, []}, - {maintainers, ["EMQX Team "]}, {links, [ - {"Homepage", "https://emqx.io/"}, + {"Homepage", "https://www.emqx.com/"}, {"Github", "https://github.com/emqx"} ]} ]}. diff --git a/rel/i18n/emqx_node_rebalance_api.hocon b/rel/i18n/emqx_node_rebalance_api.hocon index 51d0fa8bc..0c9e369be 100644 --- a/rel/i18n/emqx_node_rebalance_api.hocon +++ b/rel/i18n/emqx_node_rebalance_api.hocon @@ -121,10 +121,10 @@ local_status_enabled.label: """Local evacuation status""" local_status_process.desc: -"""The process that is being performed on the node: evacuation or rebalance""" +"""The type of the task that is being performed on the node: 'evacuation' or 'rebalance'""" local_status_process.label: -"""Node process""" +"""Task Type""" local_status_state.desc: """The state of the process that is being performed on the node""" From 48b53b9ca448e75638b8d246e6e6be5106bbcb82 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 8 May 2023 23:25:32 +0200 Subject: [PATCH 043/144] docs: delete zh changelog --- changes/ee/feat-10075.en.md | 3 ++- changes/ee/feat-10075.zh.md | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 changes/ee/feat-10075.zh.md diff --git a/changes/ee/feat-10075.en.md b/changes/ee/feat-10075.en.md index e6e070ddc..35c3949e3 100644 --- a/changes/ee/feat-10075.en.md +++ b/changes/ee/feat-10075.en.md @@ -1 +1,2 @@ -Add node rebalance/node evacuation [functionality](https://github.com/emqx/eip/blob/main/active/0020-node-rebalance.md). +Add node rebalance/node evacuation functionality. +See also: [design doc](https://github.com/emqx/eip/blob/main/active/0020-node-rebalance.md) diff --git a/changes/ee/feat-10075.zh.md b/changes/ee/feat-10075.zh.md deleted file mode 100644 index 36c78acb8..000000000 --- a/changes/ee/feat-10075.zh.md +++ /dev/null @@ -1 +0,0 @@ -添加节点再平衡/节点疏散[功能](https://github.com/emqx/eip/blob/main/active/0020-node-rebalance.md)。 From 9e6cc0d110f17de88b1725cc10af873a9acab569 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 8 May 2023 23:28:06 +0200 Subject: [PATCH 044/144] docs: delete APL header from ee file --- .../include/emqx_node_rebalance.hrl | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lib-ee/emqx_node_rebalance/include/emqx_node_rebalance.hrl b/lib-ee/emqx_node_rebalance/include/emqx_node_rebalance.hrl index ccc671e81..7d7bc439e 100644 --- a/lib-ee/emqx_node_rebalance/include/emqx_node_rebalance.hrl +++ b/lib-ee/emqx_node_rebalance/include/emqx_node_rebalance.hrl @@ -1,17 +1,5 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. %%-------------------------------------------------------------------- -define(DEFAULT_CONN_EVICT_RATE, 500). From 3ff04d51bc5c427aea0e00d1d21359996c2613bd Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 9 May 2023 00:09:16 +0200 Subject: [PATCH 045/144] docs: refine zh tr --- rel/i18n/emqx_node_rebalance_api.hocon | 8 +- rel/i18n/zh/emqx_node_rebalance_api.hocon | 133 +++++++++++----------- 2 files changed, 70 insertions(+), 71 deletions(-) diff --git a/rel/i18n/emqx_node_rebalance_api.hocon b/rel/i18n/emqx_node_rebalance_api.hocon index 0c9e369be..bb67f2aad 100644 --- a/rel/i18n/emqx_node_rebalance_api.hocon +++ b/rel/i18n/emqx_node_rebalance_api.hocon @@ -67,25 +67,25 @@ sess_evict_rate.label: """Session eviction rate""" abs_conn_threshold.desc: -"""Maximum desired difference between the number of connections on the node and the average number of connections on the recipient nodes""" +"""Maximum desired difference between the number of connections on the node and the average number of connections on the recipient nodes. Difference lower than this is the goal of the rebalance process.""" abs_conn_threshold.label: """Absolute connection threshold""" rel_conn_threshold.desc: -"""Maximum desired fraction between the number of connections on the node and the average number of connections on the recipient nodes""" +"""Maximum desired fraction between the number of connections on the node and the average number of connections on the recipient nodes. Fraction lower than this is the goal of the rebalance process.""" rel_conn_threshold.label: """Relative connection threshold""" abs_sess_threshold.desc: -"""Maximum desired difference between the number of sessions on the node and the average number of sessions on the recipient nodes""" +"""Maximum desired difference between the number of sessions on the node and the average number of sessions on the recipient nodes. Difference lower than this is the goal of the evacuation process.""" abs_sess_threshold.label: """Absolute session threshold""" rel_sess_threshold.desc: -"""Maximum desired fraction between the number of sessions on the node and the average number of sessions on the recipient nodes""" +"""Maximum desired fraction between the number of sessions on the node and the average number of sessions on the recipient nodes. Fraction lower than this is the goal of the evacuation process""" rel_sess_threshold.label: """Relative session threshold""" diff --git a/rel/i18n/zh/emqx_node_rebalance_api.hocon b/rel/i18n/zh/emqx_node_rebalance_api.hocon index 3066158b3..5f6753aff 100644 --- a/rel/i18n/zh/emqx_node_rebalance_api.hocon +++ b/rel/i18n/zh/emqx_node_rebalance_api.hocon @@ -1,46 +1,46 @@ emqx_node_rebalance_api { load_rebalance_status.desc: -"""获取当前节点的rebalance状态""" +"""获取当前节点的重平衡状态""" load_rebalance_status.label: -"""获取rebalance状态""" +"""获取重平衡状态""" load_rebalance_global_status.desc: -"""获取集群中所有rebalance/evacuation进程的状态""" +"""获取集群中所有重平衡/疏散任务的状态""" load_rebalance_global_status.label: -"""获取全局rebalance状态""" +"""获取全局重平衡状态""" load_rebalance_availability_check.desc: -"""检查节点是否正在被evacuate或rebalance""" +"""检查节点是否正在被执行重平衡或疏散""" load_rebalance_availability_check.label: """可用性检查""" load_rebalance_start.desc: -"""启动rebalance进程""" +"""启动重平衡任务""" load_rebalance_start.label: -"""启动rebalance""" +"""启动重平衡""" load_rebalance_stop.desc: -"""停止rebalance进程""" +"""停止重平衡任务""" load_rebalance_stop.label: -"""停止rebalance""" +"""停止重平衡""" load_rebalance_evacuation_start.desc: -"""启动evacuation进程""" +"""启动疏散任务""" load_rebalance_evacuation_start.label: -"""启动evacuation""" +"""启动疏散""" load_rebalance_evacuation_stop.desc: -"""停止evacuation进程""" +"""停止疏散任务""" load_rebalance_evacuation_stop.label: -"""停止evacuation""" +"""停止疏散""" param_node.desc: """节点名称""" @@ -49,49 +49,49 @@ param_node.label: """节点名称""" wait_health_check.desc: -"""启动rebalance进程前等待的时间，单位为秒""" +"""启动重平衡任务前等待的时间，单位为秒""" wait_health_check.label: """等待健康检查""" conn_evict_rate.desc: -"""逐出连接的速率，以每秒连接数表示""" +"""每秒迁出连接数""" conn_evict_rate.label: -"""连接驱逐率""" +"""迁出速率""" sess_evict_rate.desc: -"""逐出会话的速率，以每秒会话为单位""" +"""每秒迁出会话数""" sess_evict_rate.label: -"""会话驱逐率""" +"""会话迁出速率""" abs_conn_threshold.desc: -"""节点上的连接数与接收节点上的平均连接数之间的最大期望差值""" +"""当前节点上的连接数与迁入节点上的平均连接数的差值（绝对值）上限，低于该差值时停止迁移连接。""" abs_conn_threshold.label: -"""绝对连接阈值""" +"""连接数差值""" rel_conn_threshold.desc: -"""节点上的连接数与接收节点上的平均连接数之间的最大期望分数""" +"""当前节点上的连接数与迁入节点上的平均连接数的比值上限，低于该比值时停止迁移连接。""" rel_conn_threshold.label: -"""相对连接阈值""" +"""连接数比值""" abs_sess_threshold.desc: -"""节点上的会话数与接收节点上的平均会话数之间的最大期望差异""" +"""当前节点上的会话数与迁入节点上的平均会话数之间的差值（绝对值）上限，低于该差值时停止迁移会话。""" abs_sess_threshold.label: -"""绝对会话阈值""" +"""会话数差值""" rel_sess_threshold.desc: -"""节点上的会话数与接收节点上的平均会话数之间的最大期望分数""" +"""当前节点上的会话数与迁入节点上的平均会话数的比值上限，低于该比值时停止迁移会话。""" rel_sess_threshold.label: -"""相对会话阈值""" +"""会话数比值""" wait_takeover.desc: -"""开始会话疏散过程之前等待的时间，以秒为单位""" +"""开始会话疏散任务之前的等待时间，以秒为单位""" wait_takeover.label: """等待接管""" @@ -103,91 +103,91 @@ redirect_to.label: """重定向至""" migrate_to.desc: -"""将会话迁移到的节点""" +"""接受会话迁入的节点""" migrate_to.label: -"""迁移到""" +"""迁入节点""" rebalance_nodes.desc: -"""参与rebalance的节点""" +"""参与重平衡的节点""" rebalance_nodes.label: """重新平衡节点""" local_status_enabled.desc: -"""节点是否正在撤离""" +"""节点是否正在执行重平衡疏散任务""" local_status_enabled.label: -"""当地避难状况""" +"""运行状态""" local_status_process.desc: -"""正在节点上执行的过程：疏散或重新平衡""" +"""正在节点上执行的任务：'evacuation' 或 'rebalance'""" local_status_process.label: -"""节点进程""" +"""节点任务""" local_status_state.desc: -"""正在节点上执行的进程的状态""" +"""正在节点上执行的任务的状态""" local_status_state.label: """重新平衡/疏散当前状态""" local_status_coordinator_node.desc: -"""协调再平衡过程的节点""" +"""协调分配重平衡任务的节点""" local_status_coordinator_node.label: """协调节点""" local_status_connection_eviction_rate.desc: -"""逐出连接的速率，以每秒连接数表示""" +"""每秒迁出的连接数""" local_status_connection_eviction_rate.label: -"""连接驱逐率""" +"""连接迁出速率""" local_status_session_eviction_rate.desc: -"""逐出会话的速率，以每秒会话为单位""" +"""每秒迁出的会话数""" local_status_session_eviction_rate.label: -"""会话驱逐率""" +"""会话迁出速率""" local_status_connection_goal.desc: -"""节点在重新平衡/疏散过程后应该拥有的连接数""" +"""节点在重新平衡/疏散任务完成后预期拥有的连接数""" local_status_connection_goal.label: -"""连接目标""" +"""连接数目标""" local_status_session_goal.desc: -"""疏散过程后节点应有的会话数""" +"""疏散任务完成后节点预期的会话数""" local_status_session_goal.label: -"""会话目标""" +"""会话数目标""" local_status_disconnected_session_goal.desc: -"""重新平衡过程后节点应具有的断开连接的会话数""" +"""重新平衡任务完成后节点预期的无连接的会话数""" local_status_disconnected_session_goal.label: -"""断开连接的会话目标""" +"""预期无连接会话数""" local_status_session_recipients.desc: -"""会话被疏散到的节点列表""" +"""会话被迁入的节点列表""" local_status_session_recipients.label: -"""会话收件人""" +"""会话迁入节点""" local_status_recipients.desc: -"""在重新平衡期间连接/会话被疏散到的节点列表""" +"""在重新平衡期间接受连接/会话迁入的节点列表""" local_status_recipients.label: -"""收件人""" +"""接受迁入节点""" local_status_stats.desc: -"""疏散/再平衡过程的统计""" +"""疏散/重平衡的统计""" local_status_stats.label: """统计数据""" status_stats_initial_connected.desc: -"""疏散/重新平衡过程之前节点上的连接数""" +"""疏散/重新平衡任务开始之前节点上的连接数""" status_stats_initial_connected.label: """初始连接""" @@ -199,7 +199,7 @@ status_stats_current_connected.label: """当前连接""" status_stats_initial_sessions.desc: -"""疏散/重新平衡过程之前节点上的会话数""" +"""疏散/重新平衡任务开始之前节点上的会话数""" status_stats_initial_sessions.label: """初始会话""" @@ -211,52 +211,52 @@ status_stats_current_sessions.label: """当前会话""" status_stats_current_disconnected_sessions.desc: -"""节点上当前断开连接的会话数""" +"""节点上当前无连接的会话数""" status_stats_current_disconnected_sessions.label: -"""当前断开连接的会话""" +"""当前无连接会话""" coordinator_status_donors.desc: -"""正在疏散连接/会话的节点列表""" +"""正在迁出连接/会话的节点列表""" coordinator_status_donors.label: -"""捐助者""" +"""迁出节点""" coordinator_status_donor_conn_avg.desc: -"""每个供体节点的平均连接数""" +"""每个迁出节点的平均连接数""" coordinator_status_donor_conn_avg.label: -"""捐助者连接平均值""" +"""迁出节点连接平均值""" coordinator_status_donor_sess_avg.desc: -"""每个供体节点的平均会话数""" +"""每个迁出节点的平均会话数""" coordinator_status_donor_sess_avg.label: -"""平均捐助会议""" +"""迁出节点会话平均数""" coordinator_status_node.desc: -"""协调疏散/再平衡过程的节点""" +"""协调分配疏散/重平衡任务的节点""" coordinator_status_node.label: """协调节点""" evacuation_status_node.desc: -"""正在撤离的节点""" +"""正在迁出的节点""" evacuation_status_node.label: """疏散节点""" global_status_evacuations.desc: -"""正在撤离的节点列表""" +"""正在迁出的节点列表""" global_status_evacuations.label: """疏散""" global_status_rebalances.desc: -"""协调再平衡的节点列表""" +"""协调重平衡的节点列表""" global_status_rebalances.label: -"""再平衡""" +"""重平衡""" empty_response.desc: """响应为空""" @@ -264,4 +264,3 @@ empty_response.desc: empty_response.label: """空响应""" } - From 8d83dc12e71d67957a67a6db2ce042ab6dae9c71 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 May 2023 20:47:23 +0500 Subject: [PATCH 046/144] chore(rebalance): move apps from lib-ee, add READMEs --- apps/emqx_eviction_agent/BSL.txt | 94 +++++++++++++++++++ apps/emqx_eviction_agent/README.md | 35 +++++++ .../etc/emqx_eviction_agent.conf | 0 .../emqx_eviction_agent/rebar.config | 0 .../src/emqx_eviction_agent.app.src | 0 .../src/emqx_eviction_agent.appup.src | 0 .../src/emqx_eviction_agent.erl | 0 .../src/emqx_eviction_agent_api.erl | 0 .../src/emqx_eviction_agent_app.erl | 0 .../src/emqx_eviction_agent_channel.erl | 0 .../src/emqx_eviction_agent_cli.erl | 0 .../src/emqx_eviction_agent_conn_sup.erl | 0 .../src/emqx_eviction_agent_sup.erl | 0 .../proto/emqx_eviction_agent_proto_v1.erl | 0 .../test/emqx_eviction_agent_SUITE.erl | 0 .../test/emqx_eviction_agent_api_SUITE.erl | 0 .../emqx_eviction_agent_channel_SUITE.erl | 0 .../test/emqx_eviction_agent_cli_SUITE.erl | 0 .../test/emqx_eviction_agent_test_helpers.erl | 0 apps/emqx_node_rebalance/BSL.txt | 94 +++++++++++++++++++ apps/emqx_node_rebalance/README.md | 40 ++++++++ .../etc/emqx_node_rebalance.conf | 0 .../include/emqx_node_rebalance.hrl | 0 .../emqx_node_rebalance/rebar.config | 0 .../src/emqx_node_rebalance.app.src | 0 .../src/emqx_node_rebalance.appup.src | 0 .../src/emqx_node_rebalance.erl | 0 .../src/emqx_node_rebalance_agent.erl | 0 .../src/emqx_node_rebalance_api.erl | 0 .../src/emqx_node_rebalance_app.erl | 0 .../src/emqx_node_rebalance_cli.erl | 0 .../src/emqx_node_rebalance_evacuation.erl | 0 ...emqx_node_rebalance_evacuation_persist.erl | 0 .../src/emqx_node_rebalance_status.erl | 0 .../src/emqx_node_rebalance_sup.erl | 0 .../emqx_node_rebalance_api_proto_v1.erl | 0 ...mqx_node_rebalance_evacuation_proto_v1.erl | 0 .../proto/emqx_node_rebalance_proto_v1.erl | 0 .../emqx_node_rebalance_status_proto_v1.erl | 0 .../test/emqx_node_rebalance_SUITE.erl | 0 .../test/emqx_node_rebalance_agent_SUITE.erl | 0 .../test/emqx_node_rebalance_api_SUITE.erl | 0 .../test/emqx_node_rebalance_cli_SUITE.erl | 0 .../emqx_node_rebalance_evacuation_SUITE.erl | 0 ...ode_rebalance_evacuation_persist_SUITE.erl | 0 lib-ee/emqx_eviction_agent/README.md | 9 -- lib-ee/emqx_node_rebalance/README.md | 9 -- 47 files changed, 263 insertions(+), 18 deletions(-) create mode 100644 apps/emqx_eviction_agent/BSL.txt create mode 100644 apps/emqx_eviction_agent/README.md rename {lib-ee => apps}/emqx_eviction_agent/etc/emqx_eviction_agent.conf (100%) rename {lib-ee => apps}/emqx_eviction_agent/rebar.config (100%) rename {lib-ee => apps}/emqx_eviction_agent/src/emqx_eviction_agent.app.src (100%) rename {lib-ee => apps}/emqx_eviction_agent/src/emqx_eviction_agent.appup.src (100%) rename {lib-ee => apps}/emqx_eviction_agent/src/emqx_eviction_agent.erl (100%) rename {lib-ee => apps}/emqx_eviction_agent/src/emqx_eviction_agent_api.erl (100%) rename {lib-ee => apps}/emqx_eviction_agent/src/emqx_eviction_agent_app.erl (100%) rename {lib-ee => apps}/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl (100%) rename {lib-ee => apps}/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl (100%) rename {lib-ee => apps}/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl (100%) rename {lib-ee => apps}/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl (100%) rename {lib-ee => apps}/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v1.erl (100%) rename {lib-ee => apps}/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl (100%) rename {lib-ee => apps}/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl (100%) rename {lib-ee => apps}/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl (100%) rename {lib-ee => apps}/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl (100%) rename {lib-ee => apps}/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl (100%) create mode 100644 apps/emqx_node_rebalance/BSL.txt create mode 100644 apps/emqx_node_rebalance/README.md rename {lib-ee => apps}/emqx_node_rebalance/etc/emqx_node_rebalance.conf (100%) rename {lib-ee => apps}/emqx_node_rebalance/include/emqx_node_rebalance.hrl (100%) rename {lib-ee => apps}/emqx_node_rebalance/rebar.config (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/emqx_node_rebalance.app.src (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/emqx_node_rebalance.appup.src (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/emqx_node_rebalance.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/emqx_node_rebalance_api.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/emqx_node_rebalance_app.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/emqx_node_rebalance_status.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/proto/emqx_node_rebalance_api_proto_v1.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/proto/emqx_node_rebalance_evacuation_proto_v1.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/proto/emqx_node_rebalance_proto_v1.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/src/proto/emqx_node_rebalance_status_proto_v1.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl (100%) rename {lib-ee => apps}/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl (100%) delete mode 100644 lib-ee/emqx_eviction_agent/README.md delete mode 100644 lib-ee/emqx_node_rebalance/README.md diff --git a/apps/emqx_eviction_agent/BSL.txt b/apps/emqx_eviction_agent/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_eviction_agent/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_eviction_agent/README.md b/apps/emqx_eviction_agent/README.md new file mode 100644 index 000000000..943bd7d12 --- /dev/null +++ b/apps/emqx_eviction_agent/README.md @@ -0,0 +1,35 @@ +# EMQX Eviction Agent + +`emqx_eviction_agent` is a part of the node evacuation/node rebalance feature in EMQX. +It is a low-level application that encapsulates working with actual MQTT connections. + +## Application Responsibilities + +`emqx_eviction_agent` application: + +* Blocks incoming connection to the node it is running on. +* Serves as a facade for connection/session eviction operations. +* Reports blocking status via HTTP API. + +The `emqx_eviction_agent` is relatively passive and has no eviction/rebalancing logic. It allows +`emqx_node_rebalance` to perform eviction/rebalancing operations using high-level API, without having to deal with +MQTT connections directly. + +## EMQX Integration + +`emqx_eviction_agent` interacts with the following EMQX components: +* `emqx_cm` - to get the list of active MQTT connections; +* `emqx_hooks` subsystem - to block/unblock incoming connections; +* `emqx_channel` and the corresponding connection modules to perform the eviction. + +## User Facing API + +The application provided a very simple API (CLI and HTTP) to inspect the current blocking status. + +# Documentation + +The rebalancing concept is described in the corresponding [EIP](https://github.com/emqx/eip/blob/main/active/0020-node-rebalance.md). + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). diff --git a/lib-ee/emqx_eviction_agent/etc/emqx_eviction_agent.conf b/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf similarity index 100% rename from lib-ee/emqx_eviction_agent/etc/emqx_eviction_agent.conf rename to apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf diff --git a/lib-ee/emqx_eviction_agent/rebar.config b/apps/emqx_eviction_agent/rebar.config similarity index 100% rename from lib-ee/emqx_eviction_agent/rebar.config rename to apps/emqx_eviction_agent/rebar.config diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.app.src b/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src similarity index 100% rename from lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.app.src rename to apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.appup.src b/apps/emqx_eviction_agent/src/emqx_eviction_agent.appup.src similarity index 100% rename from lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.appup.src rename to apps/emqx_eviction_agent/src/emqx_eviction_agent.appup.src diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl similarity index 100% rename from lib-ee/emqx_eviction_agent/src/emqx_eviction_agent.erl rename to apps/emqx_eviction_agent/src/emqx_eviction_agent.erl diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_api.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl similarity index 100% rename from lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_api.erl rename to apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_app.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl similarity index 100% rename from lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_app.erl rename to apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl similarity index 100% rename from lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl rename to apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl similarity index 100% rename from lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl rename to apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl similarity index 100% rename from lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl rename to apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl diff --git a/lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl similarity index 100% rename from lib-ee/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl rename to apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl diff --git a/lib-ee/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v1.erl b/apps/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v1.erl similarity index 100% rename from lib-ee/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v1.erl rename to apps/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v1.erl diff --git a/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl similarity index 100% rename from lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl rename to apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl diff --git a/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl similarity index 100% rename from lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl rename to apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl diff --git a/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl similarity index 100% rename from lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl rename to apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl diff --git a/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl similarity index 100% rename from lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl rename to apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl diff --git a/lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl similarity index 100% rename from lib-ee/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl rename to apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl diff --git a/apps/emqx_node_rebalance/BSL.txt b/apps/emqx_node_rebalance/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_node_rebalance/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_node_rebalance/README.md b/apps/emqx_node_rebalance/README.md new file mode 100644 index 000000000..8a384fb5d --- /dev/null +++ b/apps/emqx_node_rebalance/README.md @@ -0,0 +1,40 @@ +# EMQX Node Rebalance + +`emqx_node_rebalance` is a part of the node evacuation/node rebalance feature in EMQX. +It implements high-level scenarios for node evacuation and rebalancing. + +## Application Responsibilities + +`emqx_node_rebalance` application's core concept is a _rebalance coordinator_. +_Rebalance сoordinator_ is an entity that implements the rebalancing logic and orchestrates the rebalancing process. +In particular, it: + +* Enables/Disables Eviction Agent on nodes. +* Sends connection/session eviction commands to Eviction Agents according to the evacuation logic. + +We have two implementations of the _rebalance coordinator_: +* `emqx_node_rebalance` - a coordinator that implements node rebalancing; +* `emqx_node_rebalance_evacuation` - a coordinator that implements node evacuation. + +## EMQX Integration + +`emqx_node_rebalance` is a high-level application that is loosely coupled with the rest of the system. +It uses Eviction Agent to perform the required operations. + +## User Facing API + +The application provides API (CLI and HTTP) to perform the following operations: +* Start/Stop rebalancing across a set of nodes or the whole cluster; +* Start/Stop evacuation of a node; +* Get the current rebalancing status of a local node. +* Get the current rebalancing status of the whole cluster. + +Also, an HTTP endpoint is provided for liveness probes. + +# Documentation + +The rebalancing concept is described in the corresponding [EIP](https://github.com/emqx/eip/blob/main/active/0020-node-rebalance.md). + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). diff --git a/lib-ee/emqx_node_rebalance/etc/emqx_node_rebalance.conf b/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf similarity index 100% rename from lib-ee/emqx_node_rebalance/etc/emqx_node_rebalance.conf rename to apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf diff --git a/lib-ee/emqx_node_rebalance/include/emqx_node_rebalance.hrl b/apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl similarity index 100% rename from lib-ee/emqx_node_rebalance/include/emqx_node_rebalance.hrl rename to apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl diff --git a/lib-ee/emqx_node_rebalance/rebar.config b/apps/emqx_node_rebalance/rebar.config similarity index 100% rename from lib-ee/emqx_node_rebalance/rebar.config rename to apps/emqx_node_rebalance/rebar.config diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.app.src b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src similarity index 100% rename from lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.app.src rename to apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.appup.src b/apps/emqx_node_rebalance/src/emqx_node_rebalance.appup.src similarity index 100% rename from lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.appup.src rename to apps/emqx_node_rebalance/src/emqx_node_rebalance.appup.src diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/src/emqx_node_rebalance.erl rename to apps/emqx_node_rebalance/src/emqx_node_rebalance.erl diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl rename to apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_api.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_api.erl rename to apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_app.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_app.erl rename to apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl rename to apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl rename to apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl rename to apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_status.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_status.erl rename to apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl diff --git a/lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl rename to apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl diff --git a/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_api_proto_v1.erl b/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_api_proto_v1.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_api_proto_v1.erl rename to apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_api_proto_v1.erl diff --git a/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_evacuation_proto_v1.erl b/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_evacuation_proto_v1.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_evacuation_proto_v1.erl rename to apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_evacuation_proto_v1.erl diff --git a/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_proto_v1.erl b/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_proto_v1.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_proto_v1.erl rename to apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_proto_v1.erl diff --git a/lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_status_proto_v1.erl b/apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_status_proto_v1.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/src/proto/emqx_node_rebalance_status_proto_v1.erl rename to apps/emqx_node_rebalance/src/proto/emqx_node_rebalance_status_proto_v1.erl diff --git a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl rename to apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl diff --git a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl rename to apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl diff --git a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl rename to apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl diff --git a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl rename to apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl diff --git a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl rename to apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl diff --git a/lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl similarity index 100% rename from lib-ee/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl rename to apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl diff --git a/lib-ee/emqx_eviction_agent/README.md b/lib-ee/emqx_eviction_agent/README.md deleted file mode 100644 index f9b8037bf..000000000 --- a/lib-ee/emqx_eviction_agent/README.md +++ /dev/null @@ -1,9 +0,0 @@ -emqx_eviction_agent -===== - -An OTP library - -Build ------ - - $ rebar3 compile diff --git a/lib-ee/emqx_node_rebalance/README.md b/lib-ee/emqx_node_rebalance/README.md deleted file mode 100644 index 2e56f62cd..000000000 --- a/lib-ee/emqx_node_rebalance/README.md +++ /dev/null @@ -1,9 +0,0 @@ -emqx_node_rebalance -===== - -An OTP library - -Build ------ - - $ rebar3 compile From 61deda3ea6c3839d30d30716cd6d72acfcbf24e0 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 May 2023 21:05:20 +0500 Subject: [PATCH 047/144] chore(rebalance): fix app metadata --- apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src b/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src index a360133f4..239d9052e 100644 --- a/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src +++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src @@ -14,9 +14,8 @@ {mod, {emqx_eviction_agent_app, []}}, {env, []}, {modules, []}, - {maintainers, ["EMQX Team "]}, {links, [ - {"Homepage", "https://emqx.io/"}, + {"Homepage", "https://www.emqx.com/"}, {"Github", "https://github.com/emqx"} ]} ]}. From 1d609dacf7279061e919328dc12279c703d2e442 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 9 May 2023 17:50:16 +0800 Subject: [PATCH 048/144] test: update api_config test --- apps/emqx_conf/test/emqx_conf_schema_tests.erl | 2 +- .../src/emqx_mgmt_api_configs.erl | 4 ---- .../test/emqx_mgmt_api_configs_SUITE.erl | 16 ++++++++-------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 3577edeab..e0aa1963d 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -83,7 +83,7 @@ log.file_handlers { } chars_limit = unlimited drop_mode_qlen = 3000 - enable = false + enable = true file = \"log/my-emqx.log\" flush_qlen = 8000 formatter = text diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index bc9aaf768..9b14c62fc 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -201,8 +201,6 @@ config(put, #{body := NewConf}, Req) -> case emqx_conf:update(Path, NewConf, ?OPTS) of {ok, #{raw_config := RawConf}} -> {200, RawConf}; - {error, {permission_denied, Reason}} -> - {403, #{code => 'UPDATE_FAILED', message => Reason}}; {error, Reason} -> {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Reason)}} end. @@ -247,8 +245,6 @@ config_reset(post, _Params, Req) -> case emqx_conf:reset(Path, ?OPTS) of {ok, _} -> {200}; - {error, {permission_denied, Reason}} -> - {403, #{code => 'REST_FAILED', message => Reason}}; {error, no_default_value} -> {400, #{code => 'NO_DEFAULT_VALUE', message => <<"No Default Value.">>}}; {error, Reason} -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 5a0116a4d..34b8ccd8f 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -99,24 +99,24 @@ t_log(_Config) -> {ok, Log} = get_config("log"), File = "log/emqx-test.log", %% update handler - Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), - Log2 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), + Log1 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"enable">>], Log, true), + Log2 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"to">>], Log1, File), {ok, #{}} = update_config(<<"log">>, Log2), {ok, Log3} = logger:get_handler_config(default), ?assertMatch(#{config := #{file := File}}, Log3), - ErrLog1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, 1), + ErrLog1 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"enable">>], Log, 1), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_config(<<"log">>, ErrLog1)), ErrLog2 = emqx_utils_maps:deep_put( - [<<"file_handlers">>, <<"default">>, <<"enabfe">>], Log, true + [<<"file">>, <<"default">>, <<"enabfe">>], Log, true ), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_config(<<"log">>, ErrLog2)), %% add new handler File1 = "log/emqx-test1.log", - Handler = emqx_utils_maps:deep_get([<<"file_handlers">>, <<"default">>], Log2), - NewLog1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"new">>], Log2, Handler), + Handler = emqx_utils_maps:deep_get([<<"file">>, <<"default">>], Log2), + NewLog1 = emqx_utils_maps:deep_put([<<"file">>, <<"new">>], Log2, Handler), NewLog2 = emqx_utils_maps:deep_put( - [<<"file_handlers">>, <<"new">>, <<"file">>], NewLog1, File1 + [<<"file">>, <<"new">>, <<"to">>], NewLog1, File1 ), {ok, #{}} = update_config(<<"log">>, NewLog2), {ok, Log4} = logger:get_handler_config(new), @@ -124,7 +124,7 @@ t_log(_Config) -> %% disable new handler Disable = emqx_utils_maps:deep_put( - [<<"file_handlers">>, <<"new">>, <<"enable">>], NewLog2, false + [<<"file">>, <<"new">>, <<"enable">>], NewLog2, false ), {ok, #{}} = update_config(<<"log">>, Disable), ?assertEqual({error, {not_found, new}}, logger:get_handler_config(new)), From 83e7b30a80f3c36af0653768c3575c1d9b09887d Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 10 May 2023 09:50:53 +0800 Subject: [PATCH 049/144] feat: deprecate listeners's authn http api --- apps/emqx_authn/src/emqx_authn.app.src | 2 +- apps/emqx_authn/src/emqx_authn_api.erl | 12 ++++++++++++ apps/emqx_authn/src/emqx_authn_user_import_api.erl | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index c1d48909c..3e0cf786e 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.18"}, + {vsn, "0.1.19"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]}, diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index de856f163..f46718842 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -228,6 +228,7 @@ schema("/listeners/:listener_id/authentication") -> 'operationId' => listener_authenticators, get => #{ tags => ?API_TAGS_SINGLE, + deprecated => true, description => ?DESC(listeners_listener_id_authentication_get), parameters => [param_listener_id()], responses => #{ @@ -239,6 +240,7 @@ schema("/listeners/:listener_id/authentication") -> }, post => #{ tags => ?API_TAGS_SINGLE, + deprecated => true, description => ?DESC(listeners_listener_id_authentication_post), parameters => [param_listener_id()], 'requestBody' => emqx_dashboard_swagger:schema_with_examples( @@ -260,6 +262,7 @@ schema("/listeners/:listener_id/authentication/:id") -> 'operationId' => listener_authenticator, get => #{ tags => ?API_TAGS_SINGLE, + deprecated => true, description => ?DESC(listeners_listener_id_authentication_id_get), parameters => [param_listener_id(), param_auth_id()], responses => #{ @@ -272,6 +275,7 @@ schema("/listeners/:listener_id/authentication/:id") -> }, put => #{ tags => ?API_TAGS_SINGLE, + deprecated => true, description => ?DESC(listeners_listener_id_authentication_id_put), parameters => [param_listener_id(), param_auth_id()], 'requestBody' => emqx_dashboard_swagger:schema_with_examples( @@ -287,6 +291,7 @@ schema("/listeners/:listener_id/authentication/:id") -> }, delete => #{ tags => ?API_TAGS_SINGLE, + deprecated => true, description => ?DESC(listeners_listener_id_authentication_id_delete), parameters => [param_listener_id(), param_auth_id()], responses => #{ @@ -300,6 +305,7 @@ schema("/listeners/:listener_id/authentication/:id/status") -> 'operationId' => listener_authenticator_status, get => #{ tags => ?API_TAGS_SINGLE, + deprecated => true, description => ?DESC(listeners_listener_id_authentication_id_status_get), parameters => [param_listener_id(), param_auth_id()], responses => #{ @@ -330,6 +336,7 @@ schema("/listeners/:listener_id/authentication/:id/position/:position") -> 'operationId' => listener_authenticator_position, put => #{ tags => ?API_TAGS_SINGLE, + deprecated => true, description => ?DESC(listeners_listener_id_authentication_id_position_put), parameters => [param_listener_id(), param_auth_id(), param_position()], responses => #{ @@ -393,6 +400,7 @@ schema("/listeners/:listener_id/authentication/:id/users") -> 'operationId' => listener_authenticator_users, post => #{ tags => ?API_TAGS_SINGLE, + deprecated => true, description => ?DESC(listeners_listener_id_authentication_id_users_post), parameters => [param_auth_id(), param_listener_id()], 'requestBody' => emqx_dashboard_swagger:schema_with_examples( @@ -410,6 +418,7 @@ schema("/listeners/:listener_id/authentication/:id/users") -> }, get => #{ tags => ?API_TAGS_SINGLE, + deprecated => true, description => ?DESC(listeners_listener_id_authentication_id_users_get), parameters => [ param_listener_id(), @@ -479,6 +488,7 @@ schema("/listeners/:listener_id/authentication/:id/users/:user_id") -> 'operationId' => listener_authenticator_user, get => #{ tags => ?API_TAGS_SINGLE, + deprecated => true, description => ?DESC(listeners_listener_id_authentication_id_users_user_id_get), parameters => [param_listener_id(), param_auth_id(), param_user_id()], responses => #{ @@ -491,6 +501,7 @@ schema("/listeners/:listener_id/authentication/:id/users/:user_id") -> }, put => #{ tags => ?API_TAGS_SINGLE, + deprecated => true, description => ?DESC(listeners_listener_id_authentication_id_users_user_id_put), parameters => [param_listener_id(), param_auth_id(), param_user_id()], 'requestBody' => emqx_dashboard_swagger:schema_with_example( @@ -508,6 +519,7 @@ schema("/listeners/:listener_id/authentication/:id/users/:user_id") -> }, delete => #{ tags => ?API_TAGS_SINGLE, + deprecated => true, description => ?DESC(listeners_listener_id_authentication_id_users_user_id_delete), parameters => [param_listener_id(), param_auth_id(), param_user_id()], responses => #{ diff --git a/apps/emqx_authn/src/emqx_authn_user_import_api.erl b/apps/emqx_authn/src/emqx_authn_user_import_api.erl index bab25bb78..86cfc6247 100644 --- a/apps/emqx_authn/src/emqx_authn_user_import_api.erl +++ b/apps/emqx_authn/src/emqx_authn_user_import_api.erl @@ -72,6 +72,7 @@ schema("/listeners/:listener_id/authentication/:id/import_users") -> 'operationId' => listener_authenticator_import_users, post => #{ tags => ?API_TAGS_SINGLE, + deprecated => true, description => ?DESC(listeners_listener_id_authentication_id_import_users_post), parameters => [emqx_authn_api:param_listener_id(), emqx_authn_api:param_auth_id()], 'requestBody' => emqx_dashboard_swagger:file_schema(filename), From 0f979b3d24b7b8b7a6fda3140f234a909486409a Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 10 May 2023 14:05:35 +0800 Subject: [PATCH 050/144] refactor(tdengine): move tdengine bridge into its own app --- apps/emqx_bridge_tdengine/docker-ct | 2 ++ apps/emqx_bridge_tdengine/rebar.config | 8 ++++++++ .../emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src | 4 ++-- .../emqx_bridge_tdengine/src/emqx_bridge_tdengine.erl | 5 +++-- .../src/emqx_bridge_tdengine_connector.erl | 2 +- .../test/emqx_bridge_tdengine_SUITE.erl | 2 +- lib-ee/emqx_ee_bridge/docker-ct | 1 - lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src | 3 ++- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 8 ++++---- lib-ee/emqx_ee_connector/rebar.config | 1 - lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src | 1 - ...e_bridge_tdengine.hocon => emqx_bridge_tdengine.hocon} | 2 +- ...dengine.hocon => emqx_bridge_tdengine_connector.hocon} | 2 +- ...e_bridge_tdengine.hocon => emqx_bridge_tdengine.hocon} | 2 +- ...dengine.hocon => emqx_bridge_tdengine_connector.hocon} | 2 +- 15 files changed, 27 insertions(+), 18 deletions(-) create mode 100644 apps/emqx_bridge_tdengine/docker-ct create mode 100644 apps/emqx_bridge_tdengine/rebar.config rename lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl => apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.erl (95%) rename lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl => apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl (99%) rename lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl => apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl (99%) rename rel/i18n/{emqx_ee_bridge_tdengine.hocon => emqx_bridge_tdengine.hocon} (97%) rename rel/i18n/{emqx_ee_connector_tdengine.hocon => emqx_bridge_tdengine_connector.hocon} (88%) rename rel/i18n/zh/{emqx_ee_bridge_tdengine.hocon => emqx_bridge_tdengine.hocon} (96%) rename rel/i18n/zh/{emqx_ee_connector_tdengine.hocon => emqx_bridge_tdengine_connector.hocon} (88%) diff --git a/apps/emqx_bridge_tdengine/docker-ct b/apps/emqx_bridge_tdengine/docker-ct new file mode 100644 index 000000000..c6f0bc826 --- /dev/null +++ b/apps/emqx_bridge_tdengine/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +tdengine diff --git a/apps/emqx_bridge_tdengine/rebar.config b/apps/emqx_bridge_tdengine/rebar.config new file mode 100644 index 000000000..72ebca1db --- /dev/null +++ b/apps/emqx_bridge_tdengine/rebar.config @@ -0,0 +1,8 @@ +{erl_opts, [debug_info]}. + +{deps, [ + {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}}, + {emqx_bridge, {path, "../../apps/emqx_bridge"}} +]}. diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src index 05e8a6f9f..141973e1e 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src @@ -1,8 +1,8 @@ {application, emqx_bridge_tdengine, [ {description, "EMQX Enterprise TDEngine Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, tdengine]}, {env, []}, {modules, []}, {links, []} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.erl similarity index 95% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl rename to apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.erl index 777bc4f2b..abdc26592 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_tdengine). +-module(emqx_bridge_tdengine). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -81,7 +81,8 @@ fields("config") -> binary(), #{desc => ?DESC("local_topic"), default => undefined} )} - ] ++ emqx_resource_schema:fields("resource_opts") ++ emqx_ee_connector_tdengine:fields(config); + ] ++ emqx_resource_schema:fields("resource_opts") ++ + emqx_bridge_tdengine_connector:fields(config); fields("post") -> [type_field(), name_field() | fields("config")]; fields("put") -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl similarity index 99% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl rename to apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl index 09cbd8db8..46a70e8b6 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_tdengine). +-module(emqx_bridge_tdengine_connector). -behaviour(emqx_resource). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl similarity index 99% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl rename to apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl index 36ed10f38..1b8db1aaa 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl +++ b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_tdengine_SUITE). +-module(emqx_bridge_tdengine_SUITE). -compile(nowarn_export_all). -compile(export_all). diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index 37a5c9765..faff109eb 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -6,6 +6,5 @@ mysql redis redis_cluster pgsql -tdengine clickhouse dynamo diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index decf7d033..df1d778fc 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -14,7 +14,8 @@ emqx_bridge_pulsar, emqx_bridge_sqlserver, emqx_bridge_rocketmq, - emqx_bridge_rabbitmq + emqx_bridge_rabbitmq, + emqx_bridge_tdengine ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 80c627cc8..9761123da 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -31,7 +31,7 @@ api_schemas(Method) -> ref(emqx_ee_bridge_redis, Method ++ "_cluster"), ref(emqx_ee_bridge_timescale, Method), ref(emqx_ee_bridge_matrix, Method), - ref(emqx_ee_bridge_tdengine, Method), + ref(emqx_bridge_tdengine, Method), ref(emqx_ee_bridge_clickhouse, Method), ref(emqx_ee_bridge_dynamo, Method), ref(emqx_bridge_rocketmq, Method), @@ -56,7 +56,7 @@ schema_modules() -> emqx_ee_bridge_pgsql, emqx_ee_bridge_timescale, emqx_ee_bridge_matrix, - emqx_ee_bridge_tdengine, + emqx_bridge_tdengine, emqx_ee_bridge_clickhouse, emqx_ee_bridge_dynamo, emqx_bridge_rocketmq, @@ -100,7 +100,7 @@ resource_type(redis_cluster) -> emqx_ee_connector_redis; resource_type(pgsql) -> emqx_connector_pgsql; resource_type(timescale) -> emqx_connector_pgsql; resource_type(matrix) -> emqx_connector_pgsql; -resource_type(tdengine) -> emqx_ee_connector_tdengine; +resource_type(tdengine) -> emqx_bridge_tdengine_connector; resource_type(clickhouse) -> emqx_ee_connector_clickhouse; resource_type(dynamo) -> emqx_ee_connector_dynamo; resource_type(rocketmq) -> emqx_bridge_rocketmq_connector; @@ -139,7 +139,7 @@ fields(bridges) -> )}, {tdengine, mk( - hoconsc:map(name, ref(emqx_ee_bridge_tdengine, "config")), + hoconsc:map(name, ref(emqx_bridge_tdengine, "config")), #{ desc => <<"TDengine Bridge Config">>, required => false diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index a61ed9630..a9c2b4181 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -2,7 +2,6 @@ {deps, [ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, - {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}}, {emqx, {path, "../../apps/emqx"}}, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index d02995975..32dcc084c 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -8,7 +8,6 @@ ecpool, hstreamdb_erl, influxdb, - tdengine, clickhouse, erlcloud ]}, diff --git a/rel/i18n/emqx_ee_bridge_tdengine.hocon b/rel/i18n/emqx_bridge_tdengine.hocon similarity index 97% rename from rel/i18n/emqx_ee_bridge_tdengine.hocon rename to rel/i18n/emqx_bridge_tdengine.hocon index e6ece89c8..2d1059d28 100644 --- a/rel/i18n/emqx_ee_bridge_tdengine.hocon +++ b/rel/i18n/emqx_bridge_tdengine.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_tdengine { +emqx_bridge_tdengine { config_enable.desc: """Enable or disable this bridge""" diff --git a/rel/i18n/emqx_ee_connector_tdengine.hocon b/rel/i18n/emqx_bridge_tdengine_connector.hocon similarity index 88% rename from rel/i18n/emqx_ee_connector_tdengine.hocon rename to rel/i18n/emqx_bridge_tdengine_connector.hocon index 9a34b32ce..9c42dbaa0 100644 --- a/rel/i18n/emqx_ee_connector_tdengine.hocon +++ b/rel/i18n/emqx_bridge_tdengine_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_tdengine { +emqx_bridge_tdengine_connector { server.desc: """The IPv4 or IPv6 address or the hostname to connect to.
diff --git a/rel/i18n/zh/emqx_ee_bridge_tdengine.hocon b/rel/i18n/zh/emqx_bridge_tdengine.hocon similarity index 96% rename from rel/i18n/zh/emqx_ee_bridge_tdengine.hocon rename to rel/i18n/zh/emqx_bridge_tdengine.hocon index 5e417a1c7..8d0c7a24e 100644 --- a/rel/i18n/zh/emqx_ee_bridge_tdengine.hocon +++ b/rel/i18n/zh/emqx_bridge_tdengine.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_tdengine { +emqx_bridge_tdengine { config_enable.desc: """启用/禁用桥接""" diff --git a/rel/i18n/zh/emqx_ee_connector_tdengine.hocon b/rel/i18n/zh/emqx_bridge_tdengine_connector.hocon similarity index 88% rename from rel/i18n/zh/emqx_ee_connector_tdengine.hocon rename to rel/i18n/zh/emqx_bridge_tdengine_connector.hocon index f3064aeb5..6465bff35 100644 --- a/rel/i18n/zh/emqx_ee_connector_tdengine.hocon +++ b/rel/i18n/zh/emqx_bridge_tdengine_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_tdengine { +emqx_bridge_tdengine_connector { server.desc: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
From cabbf0f768ed87ae92c9af80834aa44f6455da64 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 10 May 2023 14:14:15 +0800 Subject: [PATCH 051/144] chore: update changes --- changes/ee/feat-10650.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ee/feat-10650.en.md diff --git a/changes/ee/feat-10650.en.md b/changes/ee/feat-10650.en.md new file mode 100644 index 000000000..1744fc010 --- /dev/null +++ b/changes/ee/feat-10650.en.md @@ -0,0 +1 @@ +Refactor the directory structure of the TDEngine data bridge. From b80227a02aacb26158f055b765b8ab5b99652416 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 10 May 2023 14:49:34 +0800 Subject: [PATCH 052/144] fix: crash when sysmon.os.mem_check_interval = disabled --- apps/emqx/src/emqx_os_mon.erl | 17 +++--------- apps/emqx/test/emqx_os_mon_SUITE.erl | 39 ++++++++++++++++++---------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index 4810798eb..144d2bfe5 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -23,8 +23,6 @@ -export([start_link/0]). -export([ - get_mem_check_interval/0, - set_mem_check_interval/1, get_sysmem_high_watermark/0, set_sysmem_high_watermark/1, get_procmem_high_watermark/0, @@ -46,6 +44,9 @@ terminate/2, code_change/3 ]). +-ifdef(TEST). +-export([is_sysmem_check_supported/0]). +-endif. -include("emqx.hrl"). @@ -61,14 +62,6 @@ update(OS) -> %% API %%-------------------------------------------------------------------- -get_mem_check_interval() -> - memsup:get_check_interval(). - -set_mem_check_interval(Seconds) when Seconds < 60000 -> - memsup:set_check_interval(1); -set_mem_check_interval(Seconds) -> - memsup:set_check_interval(Seconds div 60000). - get_sysmem_high_watermark() -> gen_server:call(?OS_MON, ?FUNCTION_NAME, infinity). @@ -103,11 +96,9 @@ init_os_monitor() -> init_os_monitor(OS) -> #{ sysmem_high_watermark := SysHW, - procmem_high_watermark := PHW, - mem_check_interval := MCI + procmem_high_watermark := PHW } = OS, set_procmem_high_watermark(PHW), - set_mem_check_interval(MCI), ok = update_mem_alarm_status(SysHW), SysHW. diff --git a/apps/emqx/test/emqx_os_mon_SUITE.erl b/apps/emqx/test/emqx_os_mon_SUITE.erl index 0538d949a..e76928114 100644 --- a/apps/emqx/test/emqx_os_mon_SUITE.erl +++ b/apps/emqx/test/emqx_os_mon_SUITE.erl @@ -43,8 +43,8 @@ init_per_testcase(t_cpu_check_alarm, Config) -> {ok, _} = supervisor:restart_child(emqx_sys_sup, emqx_os_mon), Config; init_per_testcase(t_sys_mem_check_alarm, Config) -> - case os:type() of - {unix, linux} -> + case emqx_os_mon:is_sysmem_check_supported() of + true -> SysMon = emqx_config:get([sysmon, os], #{}), emqx_config:put([sysmon, os], SysMon#{ sysmem_high_watermark => 0.51, @@ -54,7 +54,7 @@ init_per_testcase(t_sys_mem_check_alarm, Config) -> ok = supervisor:terminate_child(emqx_sys_sup, emqx_os_mon), {ok, _} = supervisor:restart_child(emqx_sys_sup, emqx_os_mon), Config; - _ -> + false -> Config end; init_per_testcase(_, Config) -> @@ -63,12 +63,6 @@ init_per_testcase(_, Config) -> Config. t_api(_) -> - ?assertEqual(60000, emqx_os_mon:get_mem_check_interval()), - ?assertEqual(ok, emqx_os_mon:set_mem_check_interval(30000)), - ?assertEqual(60000, emqx_os_mon:get_mem_check_interval()), - ?assertEqual(ok, emqx_os_mon:set_mem_check_interval(122000)), - ?assertEqual(120000, emqx_os_mon:get_mem_check_interval()), - ?assertEqual(0.7, emqx_os_mon:get_sysmem_high_watermark()), ?assertEqual(ok, emqx_os_mon:set_sysmem_high_watermark(0.8)), ?assertEqual(0.8, emqx_os_mon:get_sysmem_high_watermark()), @@ -86,12 +80,29 @@ t_api(_) -> gen_server:stop(emqx_os_mon), ok. +t_sys_mem_check_disable(Config) -> + case emqx_os_mon:is_sysmem_check_supported() of + true -> do_sys_mem_check_disable(Config); + false -> skip + end. + +do_sys_mem_check_disable(_Config) -> + MemRef0 = maps:get(mem_time_ref, sys:get_state(emqx_os_mon)), + ?assertEqual(true, is_reference(MemRef0), MemRef0), + emqx_config:put([sysmon, os, mem_check_interval], 1000), + emqx_os_mon:update(emqx_config:get([sysmon, os])), + MemRef1 = maps:get(mem_time_ref, sys:get_state(emqx_os_mon)), + ?assertEqual(true, is_reference(MemRef1), {MemRef0, MemRef1}), + ?assertNotEqual(MemRef0, MemRef1), + emqx_config:put([sysmon, os, mem_check_interval], disabled), + emqx_os_mon:update(emqx_config:get([sysmon, os])), + ?assertEqual(undefined, maps:get(mem_time_ref, sys:get_state(emqx_os_mon))), + ok. + t_sys_mem_check_alarm(Config) -> - case os:type() of - {unix, linux} -> - do_sys_mem_check_alarm(Config); - _ -> - skip + case emqx_os_mon:is_sysmem_check_supported() of + true -> do_sys_mem_check_alarm(Config); + false -> skip end. do_sys_mem_check_alarm(_Config) -> From 898826f15f61a69cd9539a9f2334ffa004ac4f81 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 10 May 2023 14:56:56 +0800 Subject: [PATCH 053/144] chore: add changelog for 10659 --- changes/ce/fix-10659.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-10659.en.md diff --git a/changes/ce/fix-10659.en.md b/changes/ce/fix-10659.en.md new file mode 100644 index 000000000..97cf86e42 --- /dev/null +++ b/changes/ce/fix-10659.en.md @@ -0,0 +1 @@ +Fix the issue where emqx cannot start when `sysmon.os.mem_check_interval` is disabled. From 08af90daa9e27301248d860d2f05b1669ffb0020 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 10 May 2023 10:38:40 +0200 Subject: [PATCH 054/144] fix: the iotdb password field so it has the password format --- apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl index e0312bb02..90e8d18a4 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl @@ -54,6 +54,7 @@ fields(auth_basic) -> mk(binary(), #{ required => true, desc => ?DESC("config_auth_basic_password"), + format => <<"password">>, sensitive => true, converter => fun emqx_schema:password_converter/2 })} From 066ed5c6ec9f88809d27699dafa471a720d3961c Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 10 May 2023 16:51:33 +0800 Subject: [PATCH 055/144] refactor(pgsql): move pgsql && matrix && timescale bridges into their own app --- apps/emqx_bridge_matrix/rebar.config | 7 +++++++ .../emqx_bridge_matrix/src/emqx_bridge_matrix.erl | 8 ++++---- apps/emqx_bridge_pgsql/docker-ct | 2 ++ apps/emqx_bridge_pgsql/rebar.config | 7 +++++++ .../emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl | 4 ++-- .../test/emqx_bridge_pgsql_SUITE.erl | 2 +- apps/emqx_bridge_timescale/rebar.config | 7 +++++++ .../src/emqx_bridge_timescale.erl | 8 ++++---- lib-ee/emqx_ee_bridge/docker-ct | 1 - lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 14 +++++++------- ..._bridge_pgsql.hocon => emqx_bridge_pgsql.hocon} | 2 +- ..._bridge_pgsql.hocon => emqx_bridge_pgsql.hocon} | 2 +- 12 files changed, 43 insertions(+), 21 deletions(-) create mode 100644 apps/emqx_bridge_matrix/rebar.config rename lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_matrix.erl => apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl (81%) create mode 100644 apps/emqx_bridge_pgsql/docker-ct create mode 100644 apps/emqx_bridge_pgsql/rebar.config rename lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl => apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl (96%) rename lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl => apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl (99%) create mode 100644 apps/emqx_bridge_timescale/rebar.config rename lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_timescale.erl => apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl (80%) rename rel/i18n/{emqx_ee_bridge_pgsql.hocon => emqx_bridge_pgsql.hocon} (97%) rename rel/i18n/zh/{emqx_ee_bridge_pgsql.hocon => emqx_bridge_pgsql.hocon} (96%) diff --git a/apps/emqx_bridge_matrix/rebar.config b/apps/emqx_bridge_matrix/rebar.config new file mode 100644 index 000000000..87c145f26 --- /dev/null +++ b/apps/emqx_bridge_matrix/rebar.config @@ -0,0 +1,7 @@ +{erl_opts, [debug_info]}. + +{deps, [ + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}}, + {emqx_bridge, {path, "../../apps/emqx_bridge"}} +]}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_matrix.erl b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl similarity index 81% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_matrix.erl rename to apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl index 106fac48a..abd98adb6 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_matrix.erl +++ b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_matrix). +-module(emqx_bridge_matrix). -export([ conn_bridge_examples/1 @@ -22,7 +22,7 @@ conn_bridge_examples(Method) -> #{ <<"matrix">> => #{ summary => <<"Matrix Bridge">>, - value => emqx_ee_bridge_pgsql:values(Method, matrix) + value => emqx_bridge_pgsql:values(Method, matrix) } } ]. @@ -34,9 +34,9 @@ namespace() -> "bridge_matrix". roots() -> []. fields("post") -> - emqx_ee_bridge_pgsql:fields("post", matrix); + emqx_bridge_pgsql:fields("post", matrix); fields(Method) -> - emqx_ee_bridge_pgsql:fields(Method). + emqx_bridge_pgsql:fields(Method). desc(_) -> undefined. diff --git a/apps/emqx_bridge_pgsql/docker-ct b/apps/emqx_bridge_pgsql/docker-ct new file mode 100644 index 000000000..81281026b --- /dev/null +++ b/apps/emqx_bridge_pgsql/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +pgsql diff --git a/apps/emqx_bridge_pgsql/rebar.config b/apps/emqx_bridge_pgsql/rebar.config new file mode 100644 index 000000000..87c145f26 --- /dev/null +++ b/apps/emqx_bridge_pgsql/rebar.config @@ -0,0 +1,7 @@ +{erl_opts, [debug_info]}. + +{deps, [ + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}}, + {emqx_bridge, {path, "../../apps/emqx_bridge"}} +]}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl similarity index 96% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl rename to apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl index a5dcb19e6..4615b6789 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl +++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_pgsql). +-module(emqx_bridge_pgsql). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl similarity index 99% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl rename to apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl index d76149b16..9f2011779 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl +++ b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_pgsql_SUITE). +-module(emqx_bridge_pgsql_SUITE). -compile(nowarn_export_all). -compile(export_all). diff --git a/apps/emqx_bridge_timescale/rebar.config b/apps/emqx_bridge_timescale/rebar.config new file mode 100644 index 000000000..87c145f26 --- /dev/null +++ b/apps/emqx_bridge_timescale/rebar.config @@ -0,0 +1,7 @@ +{erl_opts, [debug_info]}. + +{deps, [ + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}}, + {emqx_bridge, {path, "../../apps/emqx_bridge"}} +]}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_timescale.erl b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl similarity index 80% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_timescale.erl rename to apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl index 20d940462..c4dedf07c 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_timescale.erl +++ b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_timescale). +-module(emqx_bridge_timescale). -export([ conn_bridge_examples/1 @@ -22,7 +22,7 @@ conn_bridge_examples(Method) -> #{ <<"timescale">> => #{ summary => <<"Timescale Bridge">>, - value => emqx_ee_bridge_pgsql:values(Method, timescale) + value => emqx_bridge_pgsql:values(Method, timescale) } } ]. @@ -34,9 +34,9 @@ namespace() -> "bridge_timescale". roots() -> []. fields("post") -> - emqx_ee_bridge_pgsql:fields("post", timescale); + emqx_bridge_pgsql:fields("post", timescale); fields(Method) -> - emqx_ee_bridge_pgsql:fields(Method). + emqx_bridge_pgsql:fields(Method). desc(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index faff109eb..8391d08ae 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -5,6 +5,5 @@ mongo_rs_sharded mysql redis redis_cluster -pgsql clickhouse dynamo diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 9761123da..9b3af166c 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -19,7 +19,7 @@ api_schemas(Method) -> ref(emqx_bridge_kafka, Method ++ "_producer"), ref(emqx_bridge_cassandra, Method), ref(emqx_ee_bridge_mysql, Method), - ref(emqx_ee_bridge_pgsql, Method), + ref(emqx_bridge_pgsql, Method), ref(emqx_ee_bridge_mongodb, Method ++ "_rs"), ref(emqx_ee_bridge_mongodb, Method ++ "_sharded"), ref(emqx_ee_bridge_mongodb, Method ++ "_single"), @@ -29,8 +29,8 @@ api_schemas(Method) -> ref(emqx_ee_bridge_redis, Method ++ "_single"), ref(emqx_ee_bridge_redis, Method ++ "_sentinel"), ref(emqx_ee_bridge_redis, Method ++ "_cluster"), - ref(emqx_ee_bridge_timescale, Method), - ref(emqx_ee_bridge_matrix, Method), + ref(emqx_bridge_timescale, Method), + ref(emqx_bridge_matrix, Method), ref(emqx_bridge_tdengine, Method), ref(emqx_ee_bridge_clickhouse, Method), ref(emqx_ee_bridge_dynamo, Method), @@ -53,9 +53,9 @@ schema_modules() -> emqx_ee_bridge_mongodb, emqx_ee_bridge_mysql, emqx_ee_bridge_redis, - emqx_ee_bridge_pgsql, - emqx_ee_bridge_timescale, - emqx_ee_bridge_matrix, + emqx_bridge_pgsql, + emqx_bridge_timescale, + emqx_bridge_matrix, emqx_bridge_tdengine, emqx_ee_bridge_clickhouse, emqx_ee_bridge_dynamo, @@ -280,7 +280,7 @@ pgsql_structs() -> [ {Type, mk( - hoconsc:map(name, ref(emqx_ee_bridge_pgsql, "config")), + hoconsc:map(name, ref(emqx_bridge_pgsql, "config")), #{ desc => <>, required => false diff --git a/rel/i18n/emqx_ee_bridge_pgsql.hocon b/rel/i18n/emqx_bridge_pgsql.hocon similarity index 97% rename from rel/i18n/emqx_ee_bridge_pgsql.hocon rename to rel/i18n/emqx_bridge_pgsql.hocon index 94c263a56..5295abb35 100644 --- a/rel/i18n/emqx_ee_bridge_pgsql.hocon +++ b/rel/i18n/emqx_bridge_pgsql.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_pgsql { +emqx_bridge_pgsql { config_enable.desc: """Enable or disable this bridge""" diff --git a/rel/i18n/zh/emqx_ee_bridge_pgsql.hocon b/rel/i18n/zh/emqx_bridge_pgsql.hocon similarity index 96% rename from rel/i18n/zh/emqx_ee_bridge_pgsql.hocon rename to rel/i18n/zh/emqx_bridge_pgsql.hocon index ebf7f331a..2f233d833 100644 --- a/rel/i18n/zh/emqx_ee_bridge_pgsql.hocon +++ b/rel/i18n/zh/emqx_bridge_pgsql.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_pgsql { +emqx_bridge_pgsql { config_enable.desc: """启用/禁用桥接""" From 614f9bda51f84bdf222d8205bc05956a1dae7a2f Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 10 May 2023 10:53:00 +0200 Subject: [PATCH 056/144] chore(gateway-mqttsn): fix a minor bug --- apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src | 2 +- apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src index dd48b2723..76f0f45b5 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src +++ b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_mqttsn, [ {description, "MQTT-SN Gateway"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl index 1ccc8b95a..ae1da5dac 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl @@ -2045,15 +2045,15 @@ handle_deliver( ignore_local(Delivers, Subscriber, Session, Ctx) -> Subs = emqx_session:info(subscriptions, Session), - lists:dropwhile( + lists:filter( fun({deliver, Topic, #message{from = Publisher}}) -> case maps:find(Topic, Subs) of {ok, #{nl := 1}} when Subscriber =:= Publisher -> ok = metrics_inc(Ctx, 'delivery.dropped'), ok = metrics_inc(Ctx, 'delivery.dropped.no_local'), - true; + false; _ -> - false + true end end, Delivers From 41f5eff014e475b32d96e0206416ce9cec41bcf3 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 10 May 2023 10:53:40 +0200 Subject: [PATCH 057/144] fix: issues with the RabbitMQ config --- .../src/emqx_bridge_rabbitmq.erl | 2 +- .../src/emqx_bridge_rabbitmq_connector.erl | 17 +---------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.erl index 2039fadf6..c4897fa39 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.erl +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.erl @@ -95,7 +95,7 @@ fields("config") -> fields("creation_opts") -> emqx_resource_schema:fields("creation_opts"); fields("post") -> - fields("post", clickhouse); + fields("post", rabbitmq); fields("put") -> fields("config"); fields("get") -> diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl index 6a7a68591..6f833d659 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl @@ -72,14 +72,7 @@ fields(config) -> desc => ?DESC("username") } )}, - {password, - hoconsc:mk( - typerefl:binary(), - #{ - required => true, - desc => ?DESC("password") - } - )}, + {password, fun emqx_connector_schema_lib:password/1}, {pool_size, hoconsc:mk( typerefl:pos_integer(), @@ -129,14 +122,6 @@ fields(config) -> desc => ?DESC("heartbeat") } )}, - {auto_reconnect, - hoconsc:mk( - emqx_schema:duration_ms(), - #{ - default => <<"2s">>, - desc => ?DESC("auto_reconnect") - } - )}, %% Things related to sending messages to RabbitMQ {exchange, hoconsc:mk( From f6a2f752fff1c111e7934513bb03422e347d6d8c Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 10 May 2023 17:23:26 +0800 Subject: [PATCH 058/144] chore: update changes && bump app versions --- apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src | 2 +- apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src | 2 +- apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src | 2 +- changes/ee/feat-10662.en.md | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changes/ee/feat-10662.en.md diff --git a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src index e2a17e070..7dfe7eae6 100644 --- a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src +++ b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_matrix, [ {description, "EMQX Enterprise MatrixDB Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [kernel, stdlib]}, {env, []}, diff --git a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src index c695283f3..a310b46b4 100644 --- a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src +++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_pgsql, [ {description, "EMQX Enterprise PostgreSQL Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [kernel, stdlib]}, {env, []}, diff --git a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src index 5b4431f73..f533f3b04 100644 --- a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src +++ b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_timescale, [ {description, "EMQX Enterprise TimescaleDB Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [kernel, stdlib]}, {env, []}, diff --git a/changes/ee/feat-10662.en.md b/changes/ee/feat-10662.en.md new file mode 100644 index 000000000..997a8295f --- /dev/null +++ b/changes/ee/feat-10662.en.md @@ -0,0 +1 @@ +Refactor the directory structure of the PostgreSQL && Matrix && Timescale data bridges. From 8914e006c2765ed9655689cd0949da0f7648ab5c Mon Sep 17 00:00:00 2001 From: JimMoen Date: Sat, 6 May 2023 14:12:47 +0800 Subject: [PATCH 059/144] refactor(dynamo): move dynamo bridge into its own app --- apps/emqx_bridge_dynamo/docker-ct | 2 ++ apps/emqx_bridge_dynamo/rebar.config | 11 +++++++++++ .../src/emqx_bridge_dynamo.app.src | 4 ++-- .../emqx_bridge_dynamo/src/emqx_bridge_dynamo.erl | 4 ++-- .../src/emqx_bridge_dynamo_connector.erl | 8 ++++---- .../src/emqx_bridge_dynamo_connector_client.erl | 3 ++- .../test/emqx_bridge_dynamo_SUITE.erl | 12 ++++++++++-- lib-ee/emqx_ee_bridge/docker-ct | 1 - lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src | 1 + lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 8 ++++---- lib-ee/emqx_ee_connector/rebar.config | 2 +- .../emqx_ee_connector/src/emqx_ee_connector.app.src | 3 +-- mix.exs | 1 + ..._bridge_dynamo.hocon => emqx_bridge_dynamo.hocon} | 2 +- ...namo.hocon => emqx_bridge_dynamo_connector.hocon} | 2 +- ..._bridge_dynamo.hocon => emqx_bridge_dynamo.hocon} | 2 +- ...namo.hocon => emqx_bridge_dynamo_connector.hocon} | 2 +- 17 files changed, 45 insertions(+), 23 deletions(-) create mode 100644 apps/emqx_bridge_dynamo/docker-ct create mode 100644 apps/emqx_bridge_dynamo/rebar.config rename lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl => apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.erl (97%) rename lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl => apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl (95%) rename lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl => apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl (99%) rename lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl => apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl (96%) rename rel/i18n/{emqx_ee_bridge_dynamo.hocon => emqx_bridge_dynamo.hocon} (97%) rename rel/i18n/{emqx_ee_connector_dynamo.hocon => emqx_bridge_dynamo_connector.hocon} (93%) rename rel/i18n/zh/{emqx_ee_bridge_dynamo.hocon => emqx_bridge_dynamo.hocon} (96%) rename rel/i18n/zh/{emqx_ee_connector_dynamo.hocon => emqx_bridge_dynamo_connector.hocon} (92%) diff --git a/apps/emqx_bridge_dynamo/docker-ct b/apps/emqx_bridge_dynamo/docker-ct new file mode 100644 index 000000000..b63325b8b --- /dev/null +++ b/apps/emqx_bridge_dynamo/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +dynamo diff --git a/apps/emqx_bridge_dynamo/rebar.config b/apps/emqx_bridge_dynamo/rebar.config new file mode 100644 index 000000000..fbccb5c9a --- /dev/null +++ b/apps/emqx_bridge_dynamo/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang; -*- +{erl_opts, [debug_info]}. +{deps, [ {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag, "3.5.16-emqx-1"}}} + , {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + {apps, [emqx_bridge_dynamo]} +]}. diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src index 51c717220..2d2e299d2 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src @@ -1,8 +1,8 @@ {application, emqx_bridge_dynamo, [ {description, "EMQX Enterprise Dynamo Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, erlcloud]}, {env, []}, {modules, []}, {links, []} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.erl similarity index 97% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl rename to apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.erl index cbfa5b6b1..251e79ca2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_dynamo). +-module(emqx_bridge_dynamo). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -89,7 +89,7 @@ fields("config") -> } )} ] ++ - (emqx_ee_connector_dynamo:fields(config) -- + (emqx_bridge_dynamo_connector:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); fields("creation_opts") -> emqx_resource_schema:fields("creation_opts"); diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl similarity index 95% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl rename to apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl index a17277e67..981c31090 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_dynamo). +-module(emqx_bridge_dynamo_connector). -behaviour(emqx_resource). @@ -131,7 +131,7 @@ on_batch_query(_InstanceId, Query, _State) -> on_get_status(_InstanceId, #{pool_name := Pool}) -> Health = emqx_resource_pool:health_check_workers( - Pool, {emqx_ee_connector_dynamo_client, is_connected, []} + Pool, {emqx_bridge_dynamo_connector_client, is_connected, []} ), status_result(Health). @@ -154,7 +154,7 @@ do_query( ), Result = ecpool:pick_and_do( PoolName, - {emqx_ee_connector_dynamo_client, query, [Table, Query, Templates]}, + {emqx_bridge_dynamo_connector_client, query, [Table, Query, Templates]}, no_handover ), @@ -181,7 +181,7 @@ do_query( connect(Opts) -> Options = proplists:get_value(config, Opts), - {ok, _Pid} = Result = emqx_ee_connector_dynamo_client:start_link(Options), + {ok, _Pid} = Result = emqx_bridge_dynamo_connector_client:start_link(Options), Result. parse_template(Config) -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl similarity index 99% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl rename to apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl index 8f27497fa..faaef9df4 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl @@ -1,7 +1,8 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_dynamo_client). + +-module(emqx_bridge_dynamo_connector_client). -behaviour(gen_server). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl similarity index 96% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl rename to apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl index 88bce879e..45ba6fd64 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_dynamo_SUITE). +-module(emqx_bridge_dynamo_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -24,6 +24,14 @@ -define(GET_CONFIG(KEY__, CFG__), proplists:get_value(KEY__, CFG__)). +%% How to run it locally (all commands are run in $PROJ_ROOT dir): +%% run ct in docker container +%% run script: +%% ```bash +%% ./scripts/ct/run.sh --ci --app apps/emqx_bridge_dynamo -- \ +%% --name 'test@127.0.0.1' -c -v --readable true \ +%% --suite apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl + %%------------------------------------------------------------------------------ %% CT boilerplate %%------------------------------------------------------------------------------ @@ -251,7 +259,7 @@ directly_setup_dynamo() -> directly_query(Query) -> directly_setup_dynamo(), - emqx_ee_connector_dynamo_client:execute(Query, ?TABLE_BIN). + emqx_bridge_dynamo_connector_client:execute(Query, ?TABLE_BIN). directly_get_payload(Key) -> case directly_query({get_item, {<<"id">>, Key}}) of diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index faff109eb..dce589657 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -7,4 +7,3 @@ redis redis_cluster pgsql clickhouse -dynamo diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index df1d778fc..560f1fcef 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -12,6 +12,7 @@ emqx_bridge_cassandra, emqx_bridge_opents, emqx_bridge_pulsar, + emqx_bridge_dynamo, emqx_bridge_sqlserver, emqx_bridge_rocketmq, emqx_bridge_rabbitmq, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 9761123da..8862e9693 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -33,7 +33,7 @@ api_schemas(Method) -> ref(emqx_ee_bridge_matrix, Method), ref(emqx_bridge_tdengine, Method), ref(emqx_ee_bridge_clickhouse, Method), - ref(emqx_ee_bridge_dynamo, Method), + ref(emqx_bridge_dynamo, Method), ref(emqx_bridge_rocketmq, Method), ref(emqx_bridge_sqlserver, Method), ref(emqx_bridge_opents, Method), @@ -58,7 +58,7 @@ schema_modules() -> emqx_ee_bridge_matrix, emqx_bridge_tdengine, emqx_ee_bridge_clickhouse, - emqx_ee_bridge_dynamo, + emqx_bridge_dynamo, emqx_bridge_rocketmq, emqx_bridge_sqlserver, emqx_bridge_opents, @@ -102,7 +102,7 @@ resource_type(timescale) -> emqx_connector_pgsql; resource_type(matrix) -> emqx_connector_pgsql; resource_type(tdengine) -> emqx_bridge_tdengine_connector; resource_type(clickhouse) -> emqx_ee_connector_clickhouse; -resource_type(dynamo) -> emqx_ee_connector_dynamo; +resource_type(dynamo) -> emqx_bridge_dynamo_connector; resource_type(rocketmq) -> emqx_bridge_rocketmq_connector; resource_type(sqlserver) -> emqx_bridge_sqlserver_connector; resource_type(opents) -> emqx_bridge_opents_connector; @@ -147,7 +147,7 @@ fields(bridges) -> )}, {dynamo, mk( - hoconsc:map(name, ref(emqx_ee_bridge_dynamo, "config")), + hoconsc:map(name, ref(emqx_bridge_dynamo, "config")), #{ desc => <<"Dynamo Bridge Config">>, required => false diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index a9c2b4181..e550c4b0c 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -1,9 +1,9 @@ +%% -*- mode: erlang -*- {erl_opts, [debug_info]}. {deps, [ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, - {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}}, {emqx, {path, "../../apps/emqx"}}, {emqx_utils, {path, "../../apps/emqx_utils"}} ]}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 32dcc084c..601582e82 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -8,8 +8,7 @@ ecpool, hstreamdb_erl, influxdb, - clickhouse, - erlcloud + clickhouse ]}, {env, []}, {modules, []}, diff --git a/mix.exs b/mix.exs index cafeec7bd..07e8286db 100644 --- a/mix.exs +++ b/mix.exs @@ -190,6 +190,7 @@ defmodule EMQXUmbrella.MixProject do {:snappyer, "1.2.8", override: true}, {:crc32cer, "0.1.8", override: true}, {:supervisor3, "1.1.12", override: true}, + {:erlcloud, github: "emqx/erlcloud", tag: "3.5.16-emqx-1", override: true}, {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true}, # The following two are dependencies of rabbit_common. They are needed here to # make mix not complain about conflicting versions diff --git a/rel/i18n/emqx_ee_bridge_dynamo.hocon b/rel/i18n/emqx_bridge_dynamo.hocon similarity index 97% rename from rel/i18n/emqx_ee_bridge_dynamo.hocon rename to rel/i18n/emqx_bridge_dynamo.hocon index 7725130eb..46ae9d1bb 100644 --- a/rel/i18n/emqx_ee_bridge_dynamo.hocon +++ b/rel/i18n/emqx_bridge_dynamo.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_dynamo { +emqx_bridge_dynamo { config_enable.desc: """Enable or disable this bridge""" diff --git a/rel/i18n/emqx_ee_connector_dynamo.hocon b/rel/i18n/emqx_bridge_dynamo_connector.hocon similarity index 93% rename from rel/i18n/emqx_ee_connector_dynamo.hocon rename to rel/i18n/emqx_bridge_dynamo_connector.hocon index 29b6bf99e..7c37676b5 100644 --- a/rel/i18n/emqx_ee_connector_dynamo.hocon +++ b/rel/i18n/emqx_bridge_dynamo_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_dynamo { +emqx_bridge_dynamo_connector { aws_access_key_id.desc: """Access Key ID for connecting to DynamoDB.""" diff --git a/rel/i18n/zh/emqx_ee_bridge_dynamo.hocon b/rel/i18n/zh/emqx_bridge_dynamo.hocon similarity index 96% rename from rel/i18n/zh/emqx_ee_bridge_dynamo.hocon rename to rel/i18n/zh/emqx_bridge_dynamo.hocon index adf33b9e8..6bf090c5d 100644 --- a/rel/i18n/zh/emqx_ee_bridge_dynamo.hocon +++ b/rel/i18n/zh/emqx_bridge_dynamo.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_dynamo { +emqx_bridge_dynamo { config_enable.desc: """启用/禁用桥接""" diff --git a/rel/i18n/zh/emqx_ee_connector_dynamo.hocon b/rel/i18n/zh/emqx_bridge_dynamo_connector.hocon similarity index 92% rename from rel/i18n/zh/emqx_ee_connector_dynamo.hocon rename to rel/i18n/zh/emqx_bridge_dynamo_connector.hocon index e7b911c1e..ef7ee3462 100644 --- a/rel/i18n/zh/emqx_ee_connector_dynamo.hocon +++ b/rel/i18n/zh/emqx_bridge_dynamo_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_dynamo { +emqx_bridge_dynamo_connector { aws_access_key_id.desc: """DynamoDB 的访问 ID。""" From e32ab10d012bbb702a0a580b310c015419a70d84 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 10 May 2023 14:22:56 +0800 Subject: [PATCH 060/144] chore: rename dynamo template files --- .../emqx_bridge_dynamo}/priv/dynamo/mqtt_acked.json | 0 .../emqx_bridge_dynamo}/priv/dynamo/mqtt_client.json | 0 .../emqx_bridge_dynamo}/priv/dynamo/mqtt_clientid_msg_map.json | 0 .../emqx_bridge_dynamo}/priv/dynamo/mqtt_msg.json | 0 .../emqx_bridge_dynamo}/priv/dynamo/mqtt_retain.json | 0 .../emqx_bridge_dynamo}/priv/dynamo/mqtt_sub.json | 0 .../emqx_bridge_dynamo}/priv/dynamo/mqtt_topic_msg_map.json | 0 apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl | 2 +- 8 files changed, 1 insertion(+), 1 deletion(-) rename {lib-ee/emqx_ee_bridge => apps/emqx_bridge_dynamo}/priv/dynamo/mqtt_acked.json (100%) rename {lib-ee/emqx_ee_bridge => apps/emqx_bridge_dynamo}/priv/dynamo/mqtt_client.json (100%) rename {lib-ee/emqx_ee_bridge => apps/emqx_bridge_dynamo}/priv/dynamo/mqtt_clientid_msg_map.json (100%) rename {lib-ee/emqx_ee_bridge => apps/emqx_bridge_dynamo}/priv/dynamo/mqtt_msg.json (100%) rename {lib-ee/emqx_ee_bridge => apps/emqx_bridge_dynamo}/priv/dynamo/mqtt_retain.json (100%) rename {lib-ee/emqx_ee_bridge => apps/emqx_bridge_dynamo}/priv/dynamo/mqtt_sub.json (100%) rename {lib-ee/emqx_ee_bridge => apps/emqx_bridge_dynamo}/priv/dynamo/mqtt_topic_msg_map.json (100%) diff --git a/lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_acked.json b/apps/emqx_bridge_dynamo/priv/dynamo/mqtt_acked.json similarity index 100% rename from lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_acked.json rename to apps/emqx_bridge_dynamo/priv/dynamo/mqtt_acked.json diff --git a/lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_client.json b/apps/emqx_bridge_dynamo/priv/dynamo/mqtt_client.json similarity index 100% rename from lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_client.json rename to apps/emqx_bridge_dynamo/priv/dynamo/mqtt_client.json diff --git a/lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_clientid_msg_map.json b/apps/emqx_bridge_dynamo/priv/dynamo/mqtt_clientid_msg_map.json similarity index 100% rename from lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_clientid_msg_map.json rename to apps/emqx_bridge_dynamo/priv/dynamo/mqtt_clientid_msg_map.json diff --git a/lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_msg.json b/apps/emqx_bridge_dynamo/priv/dynamo/mqtt_msg.json similarity index 100% rename from lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_msg.json rename to apps/emqx_bridge_dynamo/priv/dynamo/mqtt_msg.json diff --git a/lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_retain.json b/apps/emqx_bridge_dynamo/priv/dynamo/mqtt_retain.json similarity index 100% rename from lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_retain.json rename to apps/emqx_bridge_dynamo/priv/dynamo/mqtt_retain.json diff --git a/lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_sub.json b/apps/emqx_bridge_dynamo/priv/dynamo/mqtt_sub.json similarity index 100% rename from lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_sub.json rename to apps/emqx_bridge_dynamo/priv/dynamo/mqtt_sub.json diff --git a/lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_topic_msg_map.json b/apps/emqx_bridge_dynamo/priv/dynamo/mqtt_topic_msg_map.json similarity index 100% rename from lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_topic_msg_map.json rename to apps/emqx_bridge_dynamo/priv/dynamo/mqtt_topic_msg_map.json diff --git a/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl b/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl index 45ba6fd64..da87f6047 100644 --- a/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl +++ b/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl @@ -232,7 +232,7 @@ query_resource(Config, Request) -> ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). -%% create a table, use the lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_msg.json as template +%% create a table, use the apps/emqx_bridge_dynamo/priv/dynamo/mqtt_msg.json as template create_table(Config) -> directly_setup_dynamo(), delete_table(Config), From a32be1ab852952cdd2770a9b60bcf4d57c6b3ba9 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 10 May 2023 19:38:19 +0800 Subject: [PATCH 061/144] chore: bump erlcloud dependencies vsns --- mix.exs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mix.exs b/mix.exs index 07e8286db..caee134c8 100644 --- a/mix.exs +++ b/mix.exs @@ -191,6 +191,12 @@ defmodule EMQXUmbrella.MixProject do {:crc32cer, "0.1.8", override: true}, {:supervisor3, "1.1.12", override: true}, {:erlcloud, github: "emqx/erlcloud", tag: "3.5.16-emqx-1", override: true}, + # erlcloud's rebar.config requires rebar3 and does not support Mix, + # so it tries to fetch deps from git. We need to override this. + {:lhttpc, tag: "1.6.2", override: true}, + {:eini, "1.2.9", override: true}, + {:base16, "1.0.0", override: true}, + # end of erlcloud's deps {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true}, # The following two are dependencies of rabbit_common. They are needed here to # make mix not complain about conflicting versions From b7126257a5744353ec1bb27a40d7ae0cbe1c7898 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 11 May 2023 13:52:13 +0800 Subject: [PATCH 062/144] fix(limiter): fix an error when setting `max_conn_rate` in a listener --- .../emqx_limiter/src/emqx_limiter_schema.erl | 3 +- apps/emqx/src/emqx_listeners.erl | 49 +++++++++++-------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index a4f7d5b89..81358fdfe 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -286,7 +286,8 @@ default_client_config() -> default_bucket_config() -> #{ rate => infinity, - burst => 0 + burst => 0, + initial => 0 }. get_listener_opts(Conf) -> diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index d56e03a1b..2b80000dc 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -347,7 +347,8 @@ do_start_listener(Type, ListenerName, #{bind := ListenOn} = Opts) when Type == tcp; Type == ssl -> Id = listener_id(Type, ListenerName), - add_limiter_bucket(Id, Opts), + Limiter = limiter(Opts), + add_limiter_bucket(Id, Limiter), esockd:open( Id, ListenOn, @@ -356,7 +357,7 @@ do_start_listener(Type, ListenerName, #{bind := ListenOn} = Opts) when #{ listener => {Type, ListenerName}, zone => zone(Opts), - limiter => limiter(Opts), + limiter => Limiter, enable_authn => enable_authn(Opts) } ]} @@ -366,9 +367,10 @@ do_start_listener(Type, ListenerName, #{bind := ListenOn} = Opts) when Type == ws; Type == wss -> Id = listener_id(Type, ListenerName), - add_limiter_bucket(Id, Opts), + Limiter = limiter(Opts), + add_limiter_bucket(Id, Limiter), RanchOpts = ranch_opts(Type, ListenOn, Opts), - WsOpts = ws_opts(Type, ListenerName, Opts), + WsOpts = ws_opts(Type, ListenerName, Opts, Limiter), case Type of ws -> cowboy:start_clear(Id, RanchOpts, WsOpts); wss -> cowboy:start_tls(Id, RanchOpts, WsOpts) @@ -415,20 +417,22 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) -> Password -> [{password, str(Password)}] end ++ optional_quic_listener_opts(Opts), + Limiter = limiter(Opts), ConnectionOpts = #{ conn_callback => emqx_quic_connection, peer_unidi_stream_count => maps:get(peer_unidi_stream_count, Opts, 1), peer_bidi_stream_count => maps:get(peer_bidi_stream_count, Opts, 10), zone => zone(Opts), listener => {quic, ListenerName}, - limiter => limiter(Opts) + limiter => Limiter }, StreamOpts = #{ stream_callback => emqx_quic_stream, active => 1 }, + Id = listener_id(quic, ListenerName), - add_limiter_bucket(Id, Opts), + add_limiter_bucket(Id, Limiter), quicer:start_listener( Id, ListenOn, @@ -532,12 +536,12 @@ esockd_opts(ListenerId, Type, Opts0) -> end ). -ws_opts(Type, ListenerName, Opts) -> +ws_opts(Type, ListenerName, Opts, Limiter) -> WsPaths = [ {emqx_utils_maps:deep_get([websocket, mqtt_path], Opts, "/mqtt"), emqx_ws_connection, #{ zone => zone(Opts), listener => {Type, ListenerName}, - limiter => limiter(Opts), + limiter => Limiter, enable_authn => enable_authn(Opts) }} ], @@ -653,26 +657,29 @@ zone(Opts) -> limiter(Opts) -> emqx_limiter_schema:get_listener_opts(Opts). -add_limiter_bucket(Id, #{limiter := Limiter}) -> +add_limiter_bucket(_Id, undefined) -> + ok; +add_limiter_bucket(Id, Limiter) -> maps:fold( fun(Type, Cfg, _) -> emqx_limiter_server:add_bucket(Id, Type, Cfg) end, ok, maps:without([client], Limiter) - ); -add_limiter_bucket(_Id, _Cfg) -> - ok. + ). -del_limiter_bucket(Id, #{limiter := Limiters}) -> - lists:foreach( - fun(Type) -> - emqx_limiter_server:del_bucket(Id, Type) - end, - maps:keys(Limiters) - ); -del_limiter_bucket(_Id, _Cfg) -> - ok. +del_limiter_bucket(Id, Conf) -> + case limiter(Conf) of + undefined -> + ok; + Limiter -> + lists:foreach( + fun(Type) -> + emqx_limiter_server:del_bucket(Id, Type) + end, + maps:keys(Limiter) + ) + end. enable_authn(Opts) -> maps:get(enable_authn, Opts, true). From 7566c2519984c2e35195a9470cbc81c52872b818 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 21 Apr 2023 16:15:45 +0200 Subject: [PATCH 063/144] refactor: delete unused connector field 'base_url' --- apps/emqx_authn/src/emqx_authn.app.src | 2 +- .../src/simple_authn/emqx_authn_http.erl | 1 - apps/emqx_authz/src/emqx_authz_api_schema.erl | 1 - apps/emqx_authz/src/emqx_authz_schema.erl | 1 - .../src/schema/emqx_bridge_webhook_schema.erl | 2 +- .../src/emqx_connector_http.erl | 36 +------------------ .../test/emqx_swagger_response_SUITE.erl | 10 ++++-- rel/i18n/emqx_connector_http.hocon | 9 ----- rel/i18n/zh/emqx_connector_http.hocon | 8 ----- 9 files changed, 10 insertions(+), 60 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index c1d48909c..3e0cf786e 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.18"}, + {vsn, "0.1.19"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]}, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index eddad92a3..421af074e 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -100,7 +100,6 @@ common_fields() -> maps:to_list( maps:without( [ - base_url, pool_type ], maps:from_list(emqx_connector_http:fields(config)) diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index 4adada182..c89ac5bee 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -114,7 +114,6 @@ authz_http_common_fields() -> maps:to_list( maps:without( [ - base_url, pool_type ], maps:from_list(emqx_connector_http:fields(config)) diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index a2a7c6b52..26a22f73b 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -230,7 +230,6 @@ http_common_fields() -> maps:to_list( maps:without( [ - base_url, pool_type ], maps:from_list(connector_fields(http)) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl index 1540f77bf..8c9f54d97 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl @@ -68,7 +68,7 @@ basic_config() -> )} ] ++ webhook_creation_opts() ++ proplists:delete( - max_retries, proplists:delete(base_url, emqx_connector_http:fields(config)) + max_retries, emqx_connector_http:fields(config) ). request_config() -> diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index bb822a60a..7218258e8 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -35,19 +35,14 @@ reply_delegator/2 ]). --type url() :: emqx_http_lib:uri_map(). --reflect_type([url/0]). --typerefl_from_string({url/0, emqx_http_lib, uri_parse}). - -export([ roots/0, fields/1, desc/1, - validations/0, namespace/0 ]). --export([check_ssl_opts/2, validate_method/1, join_paths/2]). +-export([validate_method/1, join_paths/2]). -type connect_timeout() :: emqx_schema:duration() | infinity. -type pool_type() :: random | hash. @@ -69,20 +64,6 @@ roots() -> fields(config) -> [ - {base_url, - sc( - url(), - #{ - required => true, - validator => fun - (#{query := _Query}) -> - {error, "There must be no query in the base_url"}; - (_) -> - ok - end, - desc => ?DESC("base_url") - } - )}, {connect_timeout, sc( emqx_schema:duration_ms(), @@ -171,9 +152,6 @@ desc("request") -> desc(_) -> undefined. -validations() -> - [{check_ssl_opts, fun check_ssl_opts/1}]. - validate_method(M) when M =:= <<"post">>; M =:= <<"put">>; M =:= <<"get">>; M =:= <<"delete">> -> ok; validate_method(M) -> @@ -578,18 +556,6 @@ make_method(M) when M == <<"PUT">>; M == <<"put">> -> put; make_method(M) when M == <<"GET">>; M == <<"get">> -> get; make_method(M) when M == <<"DELETE">>; M == <<"delete">> -> delete. -check_ssl_opts(Conf) -> - check_ssl_opts("base_url", Conf). - -check_ssl_opts(URLFrom, Conf) -> - #{scheme := Scheme} = hocon_maps:get(URLFrom, Conf), - SSL = hocon_maps:get("ssl", Conf), - case {Scheme, maps:get(enable, SSL, false)} of - {http, false} -> true; - {https, true} -> true; - {_, _} -> false - end. - formalize_request(Method, BasePath, {Path, Headers, _Body}) when Method =:= get; Method =:= delete -> diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index a3d2b4e75..f23752653 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -29,6 +29,10 @@ -compile(nowarn_export_all). -compile(export_all). +-type url() :: emqx_http_lib:uri_map(). +-reflect_type([url/0]). +-typerefl_from_string({url/0, emqx_http_lib, uri_parse}). + all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> @@ -314,7 +318,7 @@ t_sub_fields(_Config) -> ok. t_complicated_type(_Config) -> - Path = "/ref/complicated_type", + Path = "/ref/complex_type", Object = #{ <<"content">> => #{ <<"application/json">> => @@ -633,14 +637,14 @@ schema("/error") -> } } }; -schema("/ref/complicated_type") -> +schema("/ref/complex_type") -> #{ operationId => test, post => #{ responses => #{ 200 => [ {no_neg_integer, hoconsc:mk(non_neg_integer(), #{})}, - {url, hoconsc:mk(emqx_connector_http:url(), #{})}, + {url, hoconsc:mk(url(), #{})}, {server, hoconsc:mk(emqx_schema:ip_port(), #{})}, {connect_timeout, hoconsc:mk(emqx_connector_http:connect_timeout(), #{})}, {pool_type, hoconsc:mk(emqx_connector_http:pool_type(), #{})}, diff --git a/rel/i18n/emqx_connector_http.hocon b/rel/i18n/emqx_connector_http.hocon index 70c644e33..b511d007a 100644 --- a/rel/i18n/emqx_connector_http.hocon +++ b/rel/i18n/emqx_connector_http.hocon @@ -1,14 +1,5 @@ emqx_connector_http { -base_url.desc: -"""The base URL is the URL includes only the scheme, host and port.
-When send an HTTP request, the real URL to be used is the concatenation of the base URL and the -path parameter
-For example: `http://localhost:9901/`""" - -base_url.label: -"""Base Url""" - body.desc: """HTTP request body.""" diff --git a/rel/i18n/zh/emqx_connector_http.hocon b/rel/i18n/zh/emqx_connector_http.hocon index 5d6398b2e..af7869e12 100644 --- a/rel/i18n/zh/emqx_connector_http.hocon +++ b/rel/i18n/zh/emqx_connector_http.hocon @@ -1,13 +1,5 @@ emqx_connector_http { -base_url.desc: -"""base URL 只包含host和port。
-发送HTTP请求时，真实的URL是由base URL 和 path parameter连接而成。
-示例：`http://localhost:9901/`""" - -base_url.label: -"""Base Url""" - body.desc: """HTTP请求报文主体。""" From d9f9e951ec6ee47835f37ecb718516b49e93a9de Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 11 May 2023 16:24:01 +0800 Subject: [PATCH 064/144] fix: bad listeners default ssl_options --- apps/emqx/src/emqx_schema.erl | 18 ++--- .../emqx_conf/test/emqx_conf_schema_tests.erl | 81 +++++++++++++++++++ 2 files changed, 90 insertions(+), 9 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index cba67aca4..80fc6f2ad 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2200,7 +2200,7 @@ common_ssl_opts_schema(Defaults) -> sc( binary(), #{ - default => D("cacertfile"), + default => cert_file("cacert.pem"), required => false, desc => ?DESC(common_ssl_opts_schema_cacertfile) } @@ -2209,7 +2209,7 @@ common_ssl_opts_schema(Defaults) -> sc( binary(), #{ - default => D("certfile"), + default => cert_file("cert.pem"), required => false, desc => ?DESC(common_ssl_opts_schema_certfile) } @@ -2218,7 +2218,7 @@ common_ssl_opts_schema(Defaults) -> sc( binary(), #{ - default => D("keyfile"), + default => cert_file("key.pem"), required => false, desc => ?DESC(common_ssl_opts_schema_keyfile) } @@ -3251,13 +3251,10 @@ default_listener(ws) -> }; default_listener(SSLListener) -> %% The env variable is resolved in emqx_tls_lib by calling naive_env_interpolate - CertFile = fun(Name) -> - iolist_to_binary("${EMQX_ETC_DIR}/" ++ filename:join(["certs", Name])) - end, SslOptions = #{ - <<"cacertfile">> => CertFile(<<"cacert.pem">>), - <<"certfile">> => CertFile(<<"cert.pem">>), - <<"keyfile">> => CertFile(<<"key.pem">>) + <<"cacertfile">> => cert_file(<<"cacert.pem">>), + <<"certfile">> => cert_file(<<"cert.pem">>), + <<"keyfile">> => cert_file(<<"key.pem">>) }, case SSLListener of ssl -> @@ -3374,3 +3371,6 @@ ensure_default_listener(#{<<"default">> := _} = Map, _ListenerType) -> ensure_default_listener(Map, ListenerType) -> NewMap = Map#{<<"default">> => default_listener(ListenerType)}, keep_default_tombstone(NewMap, #{}). + +cert_file(File) -> + iolist_to_binary(filename:join(["${EMQX_ETC_DIR}", "certs", File])). diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 667d1766f..79fe30293 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -116,6 +116,87 @@ authn_validations_test() -> ), ok. +%% erlfmt-ignore +-define(LISTENERS, + """ + listeners.ssl.default.bind = 9999 + listeners.wss.default.bind = 9998 + listeners.wss.default.ssl_options.cacertfile = \"mytest/certs/cacert.pem\" + listeners.wss.new.bind = 9997 + listeners.wss.new.websocket.mqtt_path = \"/my-mqtt\" + """ +). + +listeners_test() -> + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + + Conf = <>, + {ok, ConfMap0} = hocon:binary(Conf, #{format => richmap}), + {_, ConfMap} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap0, #{format => richmap}), + #{<<"listeners">> := Listeners} = hocon_util:richmap_to_map(ConfMap), + #{ + <<"tcp">> := #{<<"default">> := Tcp}, + <<"ws">> := #{<<"default">> := Ws}, + <<"wss">> := #{<<"default">> := DefaultWss, <<"new">> := NewWss}, + <<"ssl">> := #{<<"default">> := Ssl} + } = Listeners, + DefaultCacertFile = <<"${EMQX_ETC_DIR}/certs/cacert.pem">>, + DefaultCertFile = <<"${EMQX_ETC_DIR}/certs/cert.pem">>, + DefaultKeyFile = <<"${EMQX_ETC_DIR}/certs/key.pem">>, + ?assertMatch( + #{ + <<"bind">> := {{0, 0, 0, 0}, 1883}, + <<"enabled">> := true + }, + Tcp + ), + ?assertMatch( + #{ + <<"bind">> := {{0, 0, 0, 0}, 8083}, + <<"enabled">> := true, + <<"websocket">> := #{<<"mqtt_path">> := "/mqtt"} + }, + Ws + ), + ?assertMatch( + #{ + <<"bind">> := 9999, + <<"ssl_options">> := #{ + <<"cacertfile">> := DefaultCacertFile, + <<"certfile">> := DefaultCertFile, + <<"keyfile">> := DefaultKeyFile + } + }, + Ssl + ), + ?assertMatch( + #{ + <<"bind">> := 9998, + <<"websocket">> := #{<<"mqtt_path">> := "/mqtt"}, + <<"ssl_options">> := + #{ + <<"cacertfile">> := <<"mytest/certs/cacert.pem">>, + <<"certfile">> := DefaultCertFile, + <<"keyfile">> := DefaultKeyFile + } + }, + DefaultWss + ), + ?assertMatch( + #{ + <<"bind">> := 9997, + <<"websocket">> := #{<<"mqtt_path">> := "/my-mqtt"}, + <<"ssl_options">> := + #{ + <<"cacertfile">> := DefaultCacertFile, + <<"certfile">> := DefaultCertFile, + <<"keyfile">> := DefaultKeyFile + } + }, + NewWss + ), + ok. + doc_gen_test() -> %% the json file too large to encode. { From 78c2c53daa8b552b691f74b0fb2919f1dab654f0 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 10 May 2023 18:43:57 +0800 Subject: [PATCH 065/144] fix: authn init is empty --- apps/emqx_authn/src/emqx_authn_app.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index 44fec2363..365535b77 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -23,7 +23,9 @@ %% Application callbacks -export([ start/2, - stop/1 + stop/1, + chain_configs/0, + initialize/0 ]). -include_lib("emqx/include/emqx_authentication.hrl"). @@ -54,6 +56,7 @@ stop(_State) -> initialize() -> ok = ?AUTHN:register_providers(emqx_authn:providers()), + io:format("init:~p~n", [chain_configs()]), lists:foreach( fun({ChainName, AuthConfig}) -> ?AUTHN:initialize_authentication( @@ -72,7 +75,7 @@ chain_configs() -> [global_chain_config() | listener_chain_configs()]. global_chain_config() -> - {?GLOBAL, emqx:get_config([?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY], [])}. + {?GLOBAL, emqx:get_config([?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM], [])}. listener_chain_configs() -> lists:map( From 05ac7f4a857e97a43db27aaf6ce9a842ee24d4f8 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Sat, 6 May 2023 17:20:05 +0800 Subject: [PATCH 066/144] chore: upgarde rulesql to 0.1.6 to fix invaid utf8 input --- mix.exs | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 390982eed..b9ab2db6b 100644 --- a/mix.exs +++ b/mix.exs @@ -65,7 +65,7 @@ defmodule EMQXUmbrella.MixProject do # maybe forbid to fetch quicer {:emqtt, github: "emqx/emqtt", tag: "1.8.5", override: true, system_env: maybe_no_quic_env()}, - {:rulesql, github: "emqx/rulesql", tag: "0.1.5"}, + {:rulesql, github: "emqx/rulesql", tag: "0.1.6"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, {:telemetry, "1.1.0"}, diff --git a/rebar.config b/rebar.config index 5db33f213..0c9a79e24 100644 --- a/rebar.config +++ b/rebar.config @@ -70,7 +70,7 @@ , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}} - , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}} + , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.6"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} From 3d41449fde6df7d7a92ea23bf0f3128470c5ee9c Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 11 May 2023 17:33:18 +0800 Subject: [PATCH 067/144] fix: only fill cerf_file default in server side --- apps/emqx/src/emqx_schema.erl | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 80fc6f2ad..1779457e1 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2189,8 +2189,8 @@ filter(Opts) -> %% @private This function defines the SSL opts which are commonly used by %% SSL listener and client. --spec common_ssl_opts_schema(map()) -> hocon_schema:field_schema(). -common_ssl_opts_schema(Defaults) -> +-spec common_ssl_opts_schema(map(), server | client) -> hocon_schema:field_schema(). +common_ssl_opts_schema(Defaults, Type) -> D = fun(Field) -> maps:get(to_atom(Field), Defaults, undefined) end, Df = fun(Field, Default) -> maps:get(to_atom(Field), Defaults, Default) end, Collection = maps:get(versions, Defaults, tls_all_available), @@ -2200,7 +2200,7 @@ common_ssl_opts_schema(Defaults) -> sc( binary(), #{ - default => cert_file("cacert.pem"), + default => cert_file("cacert.pem", Type), required => false, desc => ?DESC(common_ssl_opts_schema_cacertfile) } @@ -2209,7 +2209,7 @@ common_ssl_opts_schema(Defaults) -> sc( binary(), #{ - default => cert_file("cert.pem"), + default => cert_file("cert.pem", Type), required => false, desc => ?DESC(common_ssl_opts_schema_certfile) } @@ -2218,7 +2218,7 @@ common_ssl_opts_schema(Defaults) -> sc( binary(), #{ - default => cert_file("key.pem"), + default => cert_file("key.pem", Type), required => false, desc => ?DESC(common_ssl_opts_schema_keyfile) } @@ -2305,7 +2305,7 @@ common_ssl_opts_schema(Defaults) -> server_ssl_opts_schema(Defaults, IsRanchListener) -> D = fun(Field) -> maps:get(to_atom(Field), Defaults, undefined) end, Df = fun(Field, Default) -> maps:get(to_atom(Field), Defaults, Default) end, - common_ssl_opts_schema(Defaults) ++ + common_ssl_opts_schema(Defaults, server) ++ [ {"dhfile", sc( @@ -2431,7 +2431,7 @@ crl_outer_validator(_SSLOpts) -> %% @doc Make schema for SSL client. -spec client_ssl_opts_schema(map()) -> hocon_schema:field_schema(). client_ssl_opts_schema(Defaults) -> - common_ssl_opts_schema(Defaults) ++ + common_ssl_opts_schema(Defaults, client) ++ [ {"enable", sc( @@ -3252,9 +3252,9 @@ default_listener(ws) -> default_listener(SSLListener) -> %% The env variable is resolved in emqx_tls_lib by calling naive_env_interpolate SslOptions = #{ - <<"cacertfile">> => cert_file(<<"cacert.pem">>), - <<"certfile">> => cert_file(<<"cert.pem">>), - <<"keyfile">> => cert_file(<<"key.pem">>) + <<"cacertfile">> => cert_file(<<"cacert.pem">>, server), + <<"certfile">> => cert_file(<<"cert.pem">>, server), + <<"keyfile">> => cert_file(<<"key.pem">>, server) }, case SSLListener of ssl -> @@ -3372,5 +3372,5 @@ ensure_default_listener(Map, ListenerType) -> NewMap = Map#{<<"default">> => default_listener(ListenerType)}, keep_default_tombstone(NewMap, #{}). -cert_file(File) -> - iolist_to_binary(filename:join(["${EMQX_ETC_DIR}", "certs", File])). +cert_file(_File, client) -> undefined; +cert_file(File, server) -> iolist_to_binary(filename:join(["${EMQX_ETC_DIR}", "certs", File])). From 6056b0e7a8f2ecbd0dfded4fcb1118cb93392dac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Wed, 10 May 2023 22:58:59 +0800 Subject: [PATCH 068/144] fix: ensure atom key for emqx_config:get --- apps/emqx_authn/src/emqx_authn_app.erl | 13 ++++++------- apps/emqx_bridge/src/emqx_bridge_api.erl | 5 +++-- apps/emqx_bridge/src/emqx_bridge_resource.erl | 16 ++++++++++++---- .../emqx_bridge_kafka_impl_producer_SUITE.erl | 4 ++-- .../src/emqx_rule_engine.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.erl | 5 ++++- 6 files changed, 28 insertions(+), 17 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index 365535b77..5d4be5f41 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -23,9 +23,7 @@ %% Application callbacks -export([ start/2, - stop/1, - chain_configs/0, - initialize/0 + stop/1 ]). -include_lib("emqx/include/emqx_authentication.hrl"). @@ -56,7 +54,6 @@ stop(_State) -> initialize() -> ok = ?AUTHN:register_providers(emqx_authn:providers()), - io:format("init:~p~n", [chain_configs()]), lists:foreach( fun({ChainName, AuthConfig}) -> ?AUTHN:initialize_authentication( @@ -86,9 +83,11 @@ listener_chain_configs() -> ). auth_config_path(ListenerID) -> - [<<"listeners">>] ++ - binary:split(atom_to_binary(ListenerID), <<":">>) ++ - [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY]. + Names = [ + binary_to_existing_atom(N, utf8) + || N <- binary:split(atom_to_binary(ListenerID), <<":">>) + ], + [listeners] ++ Names ++ [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM]. provider_types() -> lists:map(fun({Type, _Module}) -> Type end, emqx_authn:providers()). diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 09d1159bd..c7e48990b 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -54,13 +54,14 @@ -define(BRIDGE_NOT_FOUND(BRIDGE_TYPE, BRIDGE_NAME), ?NOT_FOUND( - <<"Bridge lookup failed: bridge named '", (BRIDGE_NAME)/binary, "' of type ", + <<"Bridge lookup failed: bridge named '", (bin(BRIDGE_NAME))/binary, "' of type ", (bin(BRIDGE_TYPE))/binary, " does not exist.">> ) ). +%% Don't turn bridge_name to atom, it's maybe not a existing atom. -define(TRY_PARSE_ID(ID, EXPR), - try emqx_bridge_resource:parse_bridge_id(Id) of + try emqx_bridge_resource:parse_bridge_id(Id, #{atom_name => false}) of {BridgeType, BridgeName} -> EXPR catch diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index a8dd76214..0d2feef83 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -25,6 +25,7 @@ resource_id/2, bridge_id/2, parse_bridge_id/1, + parse_bridge_id/2, bridge_hookpoint/1, bridge_hookpoint_to_bridge_id/1 ]). @@ -86,11 +87,15 @@ bridge_id(BridgeType, BridgeName) -> Type = bin(BridgeType), <>. --spec parse_bridge_id(list() | binary() | atom()) -> {atom(), binary()}. parse_bridge_id(BridgeId) -> + parse_bridge_id(BridgeId, #{atom_name => true}). + +-spec parse_bridge_id(list() | binary() | atom(), #{atom_name => boolean()}) -> + {atom(), atom() | binary()}. +parse_bridge_id(BridgeId, Opts) -> case string:split(bin(BridgeId), ":", all) of [Type, Name] -> - {to_type_atom(Type), validate_name(Name)}; + {to_type_atom(Type), validate_name(Name, Opts)}; _ -> invalid_data( <<"should be of pattern {type}:{name}, but got ", BridgeId/binary>> @@ -105,13 +110,16 @@ bridge_hookpoint_to_bridge_id(?BRIDGE_HOOKPOINT(BridgeId)) -> bridge_hookpoint_to_bridge_id(_) -> {error, bad_bridge_hookpoint}. -validate_name(Name0) -> +validate_name(Name0, Opts) -> Name = unicode:characters_to_list(Name0, utf8), case is_list(Name) andalso Name =/= [] of true -> case lists:all(fun is_id_char/1, Name) of true -> - Name0; + case maps:get(atom_name, Opts, true) of + true -> list_to_existing_atom(Name); + false -> Name0 + end; false -> invalid_data(<<"bad name: ", Name0/binary>>) end; diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index a2111b1a8..d1a29fffe 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -583,7 +583,7 @@ config(Args0, More) -> ct:pal("Running tests with conf:\n~p", [Conf]), InstId = maps:get("instance_id", Args), <<"bridge:", BridgeId/binary>> = InstId, - {Type, Name} = emqx_bridge_resource:parse_bridge_id(BridgeId), + {Type, Name} = emqx_bridge_resource:parse_bridge_id(BridgeId, #{atom_name => false}), TypeBin = atom_to_binary(Type), hocon_tconf:check_plain( emqx_bridge_schema, @@ -596,7 +596,7 @@ config(Args0, More) -> hocon_config(Args) -> InstId = maps:get("instance_id", Args), <<"bridge:", BridgeId/binary>> = InstId, - {_Type, Name} = emqx_bridge_resource:parse_bridge_id(BridgeId), + {_Type, Name} = emqx_bridge_resource:parse_bridge_id(BridgeId, #{atom_name => false}), AuthConf = maps:get("authentication", Args), AuthTemplate = iolist_to_binary(hocon_config_template_authentication(AuthConf)), AuthConfRendered = bbmustache:render(AuthTemplate, AuthConf), diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 932ebc5ed..94a48fb35 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.15"}, + {vsn, "5.0.16"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index ada52c5aa..9dd94970b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -341,7 +341,10 @@ get_basic_usage_info() -> tally_referenced_bridges(BridgeIDs, Acc0) -> lists:foldl( fun(BridgeID, Acc) -> - {BridgeType, _BridgeName} = emqx_bridge_resource:parse_bridge_id(BridgeID), + {BridgeType, _BridgeName} = emqx_bridge_resource:parse_bridge_id( + BridgeID, + #{atom_name => false} + ), maps:update_with( BridgeType, fun(X) -> X + 1 end, From 91f97f6c29a3e7e490cc1197f8d6f7b0ffd7fa92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Thu, 11 May 2023 21:26:23 +0800 Subject: [PATCH 069/144] fix: ocsp cache SUITE failed --- apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index 75c41b9fb..b0ba4f0e2 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -967,20 +967,11 @@ do_t_validations(_Config) -> {error, {_, _, ResRaw3}} = update_listener_via_api(ListenerId, ListenerData3), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw3} = emqx_utils_json:decode(ResRaw3, [return_maps]), + %% we can't remove certfile now, because it has default value. ?assertMatch( - #{ - <<"mismatches">> := - #{ - <<"listeners:ssl_not_required_bind">> := - #{ - <<"reason">> := - <<"Server certificate must be defined when using OCSP stapling">> - } - } - }, - emqx_utils_json:decode(MsgRaw3, [return_maps]) + <<"{bad_ssl_config,#{file_read => enoent,pem_check => invalid_pem", _/binary>>, + MsgRaw3 ), - ok. t_unknown_error_fetching_ocsp_response(_Config) -> From 49e9ace1c148f623f38c24955b78754aa595e477 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 11 May 2023 22:37:50 +0500 Subject: [PATCH 070/144] fix(api): respond 404 on the deletion of nonexistent rule --- .../src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine_api.erl | 27 +++++++++++-------- .../test/emqx_rule_engine_api_SUITE.erl | 9 ++++++- changes/ce/fix-10677.en.md | 1 + 4 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 changes/ce/fix-10677.en.md diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 932ebc5ed..94a48fb35 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.15"}, + {vsn, "5.0.16"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index d66f2c1c9..fdd19bf41 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -407,17 +407,22 @@ param_path_id() -> {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} end; '/rules/:id'(delete, #{bindings := #{id := Id}}) -> - ConfPath = emqx_rule_engine:config_key_path() ++ [Id], - case emqx_conf:remove(ConfPath, #{override_to => cluster}) of - {ok, _} -> - {204}; - {error, Reason} -> - ?SLOG(error, #{ - msg => "delete_rule_failed", - id => Id, - reason => Reason - }), - {500, #{code => 'INTERNAL_ERROR', message => ?ERR_BADARGS(Reason)}} + case emqx_rule_engine:get_rule(Id) of + {ok, _Rule} -> + ConfPath = emqx_rule_engine:config_key_path() ++ [Id], + case emqx_conf:remove(ConfPath, #{override_to => cluster}) of + {ok, _} -> + {204}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "delete_rule_failed", + id => Id, + reason => Reason + }), + {500, #{code => 'INTERNAL_ERROR', message => ?ERR_BADARGS(Reason)}} + end; + not_found -> + {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}} end. '/rules/:id/metrics'(get, #{bindings := #{id := Id}}) -> diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index 8d7546fca..ccee05604 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -120,7 +120,14 @@ t_crud_rule_api(_Config) -> ) ), - %ct:pal("Show After Deleted: ~p", [NotFound]), + ?assertMatch( + {404, #{code := 'NOT_FOUND'}}, + emqx_rule_engine_api:'/rules/:id'( + delete, + #{bindings => #{id => RuleId}} + ) + ), + ?assertMatch( {404, #{code := _, message := _Message}}, emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleId}}) diff --git a/changes/ce/fix-10677.en.md b/changes/ce/fix-10677.en.md new file mode 100644 index 000000000..c669606e7 --- /dev/null +++ b/changes/ce/fix-10677.en.md @@ -0,0 +1 @@ +In Rule API, reapond with 404 HTTP error code when trying to delete a rule that does not exist. From 64dc9ed46a8e8215913298764ab156e5590b3707 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 11 May 2023 15:10:38 -0300 Subject: [PATCH 071/144] perf(metrics): avoid increasing counters by 0 Some performance tests indicate that calling `telemetry` is costly in hot paths. Since increasing a counter by 0 is a no-op, we should avoid calling `telemetry` if the amount to increase is 0. --- .../src/emqx_resource_metrics.erl | 28 +++++++++++++++++++ changes/ce/perf-10678.en.md | 1 + 2 files changed, 29 insertions(+) create mode 100644 changes/ce/perf-10678.en.md diff --git a/apps/emqx_resource/src/emqx_resource_metrics.erl b/apps/emqx_resource/src/emqx_resource_metrics.erl index 28507e291..df28d893b 100644 --- a/apps/emqx_resource/src/emqx_resource_metrics.erl +++ b/apps/emqx_resource/src/emqx_resource_metrics.erl @@ -206,6 +206,8 @@ inflight_get(ID) -> dropped_inc(ID) -> dropped_inc(ID, 1). +dropped_inc(_ID, 0) -> + ok; dropped_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, dropped], #{counter_inc => Val}, #{resource_id => ID}). @@ -216,6 +218,8 @@ dropped_get(ID) -> dropped_other_inc(ID) -> dropped_other_inc(ID, 1). +dropped_other_inc(_ID, 0) -> + ok; dropped_other_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, dropped_other], #{counter_inc => Val}, #{ resource_id => ID @@ -228,6 +232,8 @@ dropped_other_get(ID) -> dropped_expired_inc(ID) -> dropped_expired_inc(ID, 1). +dropped_expired_inc(_ID, 0) -> + ok; dropped_expired_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, dropped_expired], #{counter_inc => Val}, #{ resource_id => ID @@ -240,6 +246,8 @@ dropped_expired_get(ID) -> late_reply_inc(ID) -> late_reply_inc(ID, 1). +late_reply_inc(_ID, 0) -> + ok; late_reply_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, late_reply], #{counter_inc => Val}, #{ resource_id => ID @@ -252,6 +260,8 @@ late_reply_get(ID) -> dropped_queue_full_inc(ID) -> dropped_queue_full_inc(ID, 1). +dropped_queue_full_inc(_ID, 0) -> + ok; dropped_queue_full_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, dropped_queue_full], #{counter_inc => Val}, #{ resource_id => ID @@ -264,6 +274,8 @@ dropped_queue_full_get(ID) -> dropped_resource_not_found_inc(ID) -> dropped_resource_not_found_inc(ID, 1). +dropped_resource_not_found_inc(_ID, 0) -> + ok; dropped_resource_not_found_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, dropped_resource_not_found], #{counter_inc => Val}, #{ resource_id => ID @@ -276,6 +288,8 @@ dropped_resource_not_found_get(ID) -> dropped_resource_stopped_inc(ID) -> dropped_resource_stopped_inc(ID, 1). +dropped_resource_stopped_inc(_ID, 0) -> + ok; dropped_resource_stopped_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, dropped_resource_stopped], #{counter_inc => Val}, #{ resource_id => ID @@ -288,6 +302,8 @@ dropped_resource_stopped_get(ID) -> matched_inc(ID) -> matched_inc(ID, 1). +matched_inc(_ID, 0) -> + ok; matched_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, matched], #{counter_inc => Val}, #{resource_id => ID}). @@ -298,6 +314,8 @@ matched_get(ID) -> received_inc(ID) -> received_inc(ID, 1). +received_inc(_ID, 0) -> + ok; received_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, received], #{counter_inc => Val}, #{resource_id => ID}). @@ -308,6 +326,8 @@ received_get(ID) -> retried_inc(ID) -> retried_inc(ID, 1). +retried_inc(_ID, 0) -> + ok; retried_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, retried], #{counter_inc => Val}, #{resource_id => ID}). @@ -318,6 +338,8 @@ retried_get(ID) -> failed_inc(ID) -> failed_inc(ID, 1). +failed_inc(_ID, 0) -> + ok; failed_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, failed], #{counter_inc => Val}, #{resource_id => ID}). @@ -328,6 +350,8 @@ failed_get(ID) -> retried_failed_inc(ID) -> retried_failed_inc(ID, 1). +retried_failed_inc(_ID, 0) -> + ok; retried_failed_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, retried_failed], #{counter_inc => Val}, #{ resource_id => ID @@ -340,6 +364,8 @@ retried_failed_get(ID) -> retried_success_inc(ID) -> retried_success_inc(ID, 1). +retried_success_inc(_ID, 0) -> + ok; retried_success_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, retried_success], #{counter_inc => Val}, #{ resource_id => ID @@ -352,6 +378,8 @@ retried_success_get(ID) -> success_inc(ID) -> success_inc(ID, 1). +success_inc(_ID, 0) -> + ok; success_inc(ID, Val) -> telemetry:execute([?TELEMETRY_PREFIX, success], #{counter_inc => Val}, #{resource_id => ID}). diff --git a/changes/ce/perf-10678.en.md b/changes/ce/perf-10678.en.md new file mode 100644 index 000000000..67090cf1d --- /dev/null +++ b/changes/ce/perf-10678.en.md @@ -0,0 +1 @@ +Optimized counter increment calls to avoid work if increment is zero. From 44212f6e0667468f29d86051c1cdcb727eb21a93 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 12 May 2023 11:18:15 +0800 Subject: [PATCH 072/144] chore: add listener default changelog --- changes/ee/fix-10672.en.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ee/fix-10672.en.md diff --git a/changes/ee/fix-10672.en.md b/changes/ee/fix-10672.en.md new file mode 100644 index 000000000..cfd622701 --- /dev/null +++ b/changes/ee/fix-10672.en.md @@ -0,0 +1,2 @@ +Fix the issue where the lack of a default value for ssl_options in listeners results in startup failure. +For example, such command(`EMQX_LISTENERS__WSS__DEFAULT__BIND='0.0.0.0:8089' ./bin/emqx console`) would have caused a crash before. From 6497bcae466794b07ecbe3c589ab6627f9afeb9f Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 12 May 2023 10:47:38 +0800 Subject: [PATCH 073/144] refactor(influxdb): move influxdb bridge into its own app --- apps/emqx_bridge_influxdb/docker-ct | 2 ++ apps/emqx_bridge_influxdb/rebar.config | 8 +++++ .../src/emqx_bridge_influxdb.app.src | 2 +- .../src/emqx_bridge_influxdb.erl | 8 ++--- .../src/emqx_bridge_influxdb_connector.erl | 5 +-- .../test/emqx_bridge_influxdb_SUITE.erl | 8 ++--- .../emqx_bridge_influxdb_connector_SUITE.erl | 12 +++---- .../test/emqx_bridge_influxdb_tests.erl | 32 +++++++++++++------ lib-ee/emqx_ee_bridge/docker-ct | 1 - .../emqx_ee_bridge/src/emqx_ee_bridge.app.src | 3 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 12 +++---- .../include/emqx_ee_connector.hrl | 5 --- lib-ee/emqx_ee_connector/rebar.config | 1 - .../src/emqx_ee_connector.app.src | 1 - ...luxdb.hocon => emqx_bridge_influxdb.hocon} | 2 +- ...n => emqx_bridge_influxdb_connector.hocon} | 2 +- ...luxdb.hocon => emqx_bridge_influxdb.hocon} | 2 +- ...n => emqx_bridge_influxdb_connector.hocon} | 2 +- 18 files changed, 62 insertions(+), 46 deletions(-) create mode 100644 apps/emqx_bridge_influxdb/docker-ct create mode 100644 apps/emqx_bridge_influxdb/rebar.config rename lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl => apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl (98%) rename lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl => apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl (99%) rename lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl => apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl (99%) rename lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl => apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl (96%) rename lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl => apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_tests.erl (93%) delete mode 100644 lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl rename rel/i18n/{emqx_ee_bridge_influxdb.hocon => emqx_bridge_influxdb.hocon} (98%) rename rel/i18n/{emqx_ee_connector_influxdb.hocon => emqx_bridge_influxdb_connector.hocon} (97%) rename rel/i18n/zh/{emqx_ee_bridge_influxdb.hocon => emqx_bridge_influxdb.hocon} (98%) rename rel/i18n/zh/{emqx_ee_connector_influxdb.hocon => emqx_bridge_influxdb_connector.hocon} (97%) diff --git a/apps/emqx_bridge_influxdb/docker-ct b/apps/emqx_bridge_influxdb/docker-ct new file mode 100644 index 000000000..ef579c036 --- /dev/null +++ b/apps/emqx_bridge_influxdb/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +influxdb diff --git a/apps/emqx_bridge_influxdb/rebar.config b/apps/emqx_bridge_influxdb/rebar.config new file mode 100644 index 000000000..0b11423c4 --- /dev/null +++ b/apps/emqx_bridge_influxdb/rebar.config @@ -0,0 +1,8 @@ +{erl_opts, [debug_info]}. + +{deps, [ + {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}}, + {emqx_bridge, {path, "../../apps/emqx_bridge"}} +]}. diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src index 5443417c3..3045d907d 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src @@ -2,7 +2,7 @@ {description, "EMQX Enterprise InfluxDB Bridge"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, influxdb]}, {env, []}, {modules, []}, {links, []} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl similarity index 98% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl rename to apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl index 5693a1902..c2a04e93d 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_influxdb). +-module(emqx_bridge_influxdb). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_connector/include/emqx_connector.hrl"). @@ -134,7 +134,7 @@ influxdb_bridge_common_fields() -> emqx_resource_schema:fields("resource_opts"). connector_fields(Type) -> - emqx_ee_connector_influxdb:fields(Type). + emqx_bridge_influxdb_connector:fields(Type). type_name_fields(Type) -> [ @@ -147,9 +147,9 @@ desc("config") -> desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for InfluxDB using `", string:to_upper(Method), "` method."]; desc(influxdb_api_v1) -> - ?DESC(emqx_ee_connector_influxdb, "influxdb_api_v1"); + ?DESC(emqx_bridge_influxdb_connector, "influxdb_api_v1"); desc(influxdb_api_v2) -> - ?DESC(emqx_ee_connector_influxdb, "influxdb_api_v2"); + ?DESC(emqx_bridge_influxdb_connector, "influxdb_api_v2"); desc(_) -> undefined. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl similarity index 99% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl rename to apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl index 331577486..2f65f7902 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl @@ -1,9 +1,8 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_influxdb). +-module(emqx_bridge_influxdb_connector). --include("emqx_ee_connector.hrl"). -include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -40,6 +39,8 @@ -type ts_precision() :: ns | us | ms | s. +-define(INFLUXDB_DEFAULT_PORT, 8086). + %% influxdb servers don't need parse -define(INFLUXDB_HOST_OPTIONS, #{ default_port => ?INFLUXDB_DEFAULT_PORT diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl similarity index 99% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl rename to apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl index 6833b50c3..825721052 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl +++ b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_influxdb_SUITE). +-module(emqx_bridge_influxdb_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -583,7 +583,7 @@ t_start_already_started(Config) -> emqx_bridge_schema, InfluxDBConfigString ), ?check_trace( - emqx_ee_connector_influxdb:on_start(ResourceId, InfluxDBConfigMap), + emqx_bridge_influxdb_connector:on_start(ResourceId, InfluxDBConfigMap), fun(Result, Trace) -> ?assertMatch({ok, _}, Result), ?assertMatch([_], ?of_kind(influxdb_connector_start_already_started, Trace)), @@ -985,7 +985,7 @@ t_write_failure(Config) -> ?assertMatch([_ | _], Trace), [#{result := Result} | _] = Trace, ?assert( - not emqx_ee_connector_influxdb:is_unrecoverable_error(Result), + not emqx_bridge_influxdb_connector:is_unrecoverable_error(Result), #{got => Result} ); async -> @@ -993,7 +993,7 @@ t_write_failure(Config) -> ?assertMatch([#{action := nack} | _], Trace), [#{result := Result} | _] = Trace, ?assert( - not emqx_ee_connector_influxdb:is_unrecoverable_error(Result), + not emqx_bridge_influxdb_connector:is_unrecoverable_error(Result), #{got => Result} ) end, diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl similarity index 96% rename from lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl rename to apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl index 364821ea0..9aec94b65 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl +++ b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl @@ -2,16 +2,16 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_influxdb_SUITE). +-module(emqx_bridge_influxdb_connector_SUITE). -compile(nowarn_export_all). -compile(export_all). --include("emqx_connector.hrl"). +-include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --define(INFLUXDB_RESOURCE_MOD, emqx_ee_connector_influxdb). +-define(INFLUXDB_RESOURCE_MOD, emqx_bridge_influxdb_connector). all() -> emqx_common_test_helpers:all(?MODULE). @@ -65,7 +65,7 @@ t_lifecycle(Config) -> Host = ?config(influxdb_tcp_host, Config), Port = ?config(influxdb_tcp_port, Config), perform_lifecycle_check( - <<"emqx_ee_connector_influxdb_SUITE">>, + <<"emqx_bridge_influxdb_connector_SUITE">>, influxdb_config(Host, Port, false, <<"verify_none">>) ). @@ -124,7 +124,7 @@ perform_lifecycle_check(PoolName, InitialConfig) -> ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). t_tls_verify_none(Config) -> - PoolName = <<"emqx_ee_connector_influxdb_SUITE">>, + PoolName = <<"emqx_bridge_influxdb_connector_SUITE">>, Host = ?config(influxdb_tls_host, Config), Port = ?config(influxdb_tls_port, Config), InitialConfig = influxdb_config(Host, Port, true, <<"verify_none">>), @@ -135,7 +135,7 @@ t_tls_verify_none(Config) -> ok. t_tls_verify_peer(Config) -> - PoolName = <<"emqx_ee_connector_influxdb_SUITE">>, + PoolName = <<"emqx_bridge_influxdb_connector_SUITE">>, Host = ?config(influxdb_tls_host, Config), Port = ?config(influxdb_tls_port, Config), InitialConfig = influxdb_config(Host, Port, true, <<"verify_peer">>), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_tests.erl similarity index 93% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl rename to apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_tests.erl index 1e065f6c8..9ad685f77 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl +++ b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_tests.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_influxdb_tests). +-module(emqx_bridge_influxdb_tests). -include_lib("eunit/include/eunit.hrl"). @@ -192,7 +192,9 @@ fields => [{"field", "\"field\\4\""}], timestamp => undefined }}, - {"m5\\,mA,tag=\\=tag5\\=,\\,tag_a\\,=tag\\ 5a,tag_b=tag5b \\ field\\ =field5,field\\ _\\ a=field5a,\\,field_b\\ =\\=\\,\\ field5b ${timestamp5}", + { + "m5\\,mA,tag=\\=tag5\\=,\\,tag_a\\,=tag\\ 5a,tag_b=tag5b \\ field\\ =field5," + "field\\ _\\ a=field5a,\\,field_b\\ =\\=\\,\\ field5b ${timestamp5}", #{ measurement => "m5,mA", tags => [{"tag", "=tag5="}, {",tag_a,", "tag 5a"}, {"tag_b", "tag5b"}], @@ -200,7 +202,8 @@ {" field ", "field5"}, {"field _ a", "field5a"}, {",field_b ", "=, field5b"} ], timestamp => "${timestamp5}" - }}, + } + }, {"m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=\"field6\",field_a=\"field6a\",field_b=\"field6b\"", #{ measurement => "m6", @@ -208,20 +211,26 @@ fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], timestamp => undefined }}, - {"\\ \\ m7\\ \\ ,tag=\\ tag\\,7\\ ,tag_a=\"tag7a\",tag_b\\,tag1=tag7b field=\"field7\",field_a=field7a,field_b=\"field7b\\\\\n\"", + { + "\\ \\ m7\\ \\ ,tag=\\ tag\\,7\\ ,tag_a=\"tag7a\",tag_b\\,tag1=tag7b field=\"field7\"," + "field_a=field7a,field_b=\"field7b\\\\\n\"", #{ measurement => " m7 ", tags => [{"tag", " tag,7 "}, {"tag_a", "\"tag7a\""}, {"tag_b,tag1", "tag7b"}], fields => [{"field", "field7"}, {"field_a", "field7a"}, {"field_b", "field7b\\\n"}], timestamp => undefined - }}, - {"m8,tag=tag8,tag_a=\"tag8a\",tag_b=tag8b field=\"field8\",field_a=field8a,field_b=\"\\\"field\\\" = 8b\" ${timestamp8}", + } + }, + { + "m8,tag=tag8,tag_a=\"tag8a\",tag_b=tag8b field=\"field8\",field_a=field8a," + "field_b=\"\\\"field\\\" = 8b\" ${timestamp8}", #{ measurement => "m8", tags => [{"tag", "tag8"}, {"tag_a", "\"tag8a\""}, {"tag_b", "tag8b"}], fields => [{"field", "field8"}, {"field_a", "field8a"}, {"field_b", "\"field\" = 8b"}], timestamp => "${timestamp8}" - }}, + } + }, {"m\\9,tag=tag9,tag_a=\"tag9a\",tag_b=tag9b field\\=field=\"field9\",field_a=field9a,field_b=\"\" ${timestamp9}", #{ measurement => "m\\9", @@ -263,7 +272,9 @@ fields => [{"field", "\"field\\4\""}], timestamp => undefined }}, - {" m5\\,mA,tag=\\=tag5\\=,\\,tag_a\\,=tag\\ 5a,tag_b=tag5b \\ field\\ =field5,field\\ _\\ a=field5a,\\,field_b\\ =\\=\\,\\ field5b ${timestamp5} ", + { + " m5\\,mA,tag=\\=tag5\\=,\\,tag_a\\,=tag\\ 5a,tag_b=tag5b \\ field\\ =field5," + "field\\ _\\ a=field5a,\\,field_b\\ =\\=\\,\\ field5b ${timestamp5} ", #{ measurement => "m5,mA", tags => [{"tag", "=tag5="}, {",tag_a,", "tag 5a"}, {"tag_b", "tag5b"}], @@ -271,7 +282,8 @@ {" field ", "field5"}, {"field _ a", "field5a"}, {",field_b ", "=, field5b"} ], timestamp => "${timestamp5}" - }}, + } + }, {" m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=\"field6\",field_a=\"field6a\",field_b=\"field6b\" ", #{ measurement => "m6", @@ -330,7 +342,7 @@ to_influx_lines(RawLines) -> try %% mute error logs from this call emqx_logger:set_primary_log_level(none), - emqx_ee_bridge_influxdb:to_influx_lines(RawLines) + emqx_bridge_influxdb:to_influx_lines(RawLines) after emqx_logger:set_primary_log_level(OldLevel) end. diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index 1511687cd..de5d8c3b1 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -1,5 +1,4 @@ toxiproxy -influxdb mongo mongo_rs_sharded mysql diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 560f1fcef..6e2dbcbce 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -16,7 +16,8 @@ emqx_bridge_sqlserver, emqx_bridge_rocketmq, emqx_bridge_rabbitmq, - emqx_bridge_tdengine + emqx_bridge_tdengine, + emqx_bridge_influxdb ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 821e74fad..17ffe9b9b 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -24,8 +24,8 @@ api_schemas(Method) -> ref(emqx_ee_bridge_mongodb, Method ++ "_sharded"), ref(emqx_ee_bridge_mongodb, Method ++ "_single"), ref(emqx_ee_bridge_hstreamdb, Method), - ref(emqx_ee_bridge_influxdb, Method ++ "_api_v1"), - ref(emqx_ee_bridge_influxdb, Method ++ "_api_v2"), + ref(emqx_bridge_influxdb, Method ++ "_api_v1"), + ref(emqx_bridge_influxdb, Method ++ "_api_v2"), ref(emqx_ee_bridge_redis, Method ++ "_single"), ref(emqx_ee_bridge_redis, Method ++ "_sentinel"), ref(emqx_ee_bridge_redis, Method ++ "_cluster"), @@ -49,7 +49,7 @@ schema_modules() -> emqx_bridge_cassandra, emqx_ee_bridge_hstreamdb, emqx_bridge_gcp_pubsub, - emqx_ee_bridge_influxdb, + emqx_bridge_influxdb, emqx_ee_bridge_mongodb, emqx_ee_bridge_mysql, emqx_ee_bridge_redis, @@ -92,8 +92,8 @@ resource_type(mongodb_rs) -> emqx_ee_connector_mongodb; resource_type(mongodb_sharded) -> emqx_ee_connector_mongodb; resource_type(mongodb_single) -> emqx_ee_connector_mongodb; resource_type(mysql) -> emqx_connector_mysql; -resource_type(influxdb_api_v1) -> emqx_ee_connector_influxdb; -resource_type(influxdb_api_v2) -> emqx_ee_connector_influxdb; +resource_type(influxdb_api_v1) -> emqx_bridge_influxdb_connector; +resource_type(influxdb_api_v2) -> emqx_bridge_influxdb_connector; resource_type(redis_single) -> emqx_ee_connector_redis; resource_type(redis_sentinel) -> emqx_ee_connector_redis; resource_type(redis_cluster) -> emqx_ee_connector_redis; @@ -247,7 +247,7 @@ influxdb_structs() -> [ {Protocol, mk( - hoconsc:map(name, ref(emqx_ee_bridge_influxdb, Protocol)), + hoconsc:map(name, ref(emqx_bridge_influxdb, Protocol)), #{ desc => <<"InfluxDB Bridge Config">>, required => false diff --git a/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl b/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl deleted file mode 100644 index 4b6fbbd92..000000000 --- a/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl +++ /dev/null @@ -1,5 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%------------------------------------------------------------------- - --define(INFLUXDB_DEFAULT_PORT, 8086). diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index e550c4b0c..3414c80b5 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -2,7 +2,6 @@ {erl_opts, [debug_info]}. {deps, [ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, - {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, {emqx, {path, "../../apps/emqx"}}, {emqx_utils, {path, "../../apps/emqx_utils"}} diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 601582e82..9a4f36cf3 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -7,7 +7,6 @@ stdlib, ecpool, hstreamdb_erl, - influxdb, clickhouse ]}, {env, []}, diff --git a/rel/i18n/emqx_ee_bridge_influxdb.hocon b/rel/i18n/emqx_bridge_influxdb.hocon similarity index 98% rename from rel/i18n/emqx_ee_bridge_influxdb.hocon rename to rel/i18n/emqx_bridge_influxdb.hocon index c5cee2b66..4299f41ab 100644 --- a/rel/i18n/emqx_ee_bridge_influxdb.hocon +++ b/rel/i18n/emqx_bridge_influxdb.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_influxdb { +emqx_bridge_influxdb { config_enable.desc: """Enable or disable this bridge.""" diff --git a/rel/i18n/emqx_ee_connector_influxdb.hocon b/rel/i18n/emqx_bridge_influxdb_connector.hocon similarity index 97% rename from rel/i18n/emqx_ee_connector_influxdb.hocon rename to rel/i18n/emqx_bridge_influxdb_connector.hocon index 9c3b143a2..4169ce065 100644 --- a/rel/i18n/emqx_ee_connector_influxdb.hocon +++ b/rel/i18n/emqx_bridge_influxdb_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_influxdb { +emqx_bridge_influxdb_connector { bucket.desc: """InfluxDB bucket name.""" diff --git a/rel/i18n/zh/emqx_ee_bridge_influxdb.hocon b/rel/i18n/zh/emqx_bridge_influxdb.hocon similarity index 98% rename from rel/i18n/zh/emqx_ee_bridge_influxdb.hocon rename to rel/i18n/zh/emqx_bridge_influxdb.hocon index c9c7c6a54..350c68e39 100644 --- a/rel/i18n/zh/emqx_ee_bridge_influxdb.hocon +++ b/rel/i18n/zh/emqx_bridge_influxdb.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_influxdb { +emqx_bridge_influxdb { config_enable.desc: """启用/禁用桥接。""" diff --git a/rel/i18n/zh/emqx_ee_connector_influxdb.hocon b/rel/i18n/zh/emqx_bridge_influxdb_connector.hocon similarity index 97% rename from rel/i18n/zh/emqx_ee_connector_influxdb.hocon rename to rel/i18n/zh/emqx_bridge_influxdb_connector.hocon index 6148b400a..8477379a7 100644 --- a/rel/i18n/zh/emqx_ee_connector_influxdb.hocon +++ b/rel/i18n/zh/emqx_bridge_influxdb_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_influxdb { +emqx_bridge_influxdb_connector { bucket.desc: """InfluxDB bucket 名称。""" From 7d46fe8390e1807a76d7963bdda5a80f442b74a1 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 12 May 2023 11:46:35 +0800 Subject: [PATCH 074/144] chore: bump influxdb version && update changes --- apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src | 2 +- changes/ee/feat-10679.en.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/ee/feat-10679.en.md diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src index 3045d907d..14d881399 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_influxdb, [ {description, "EMQX Enterprise InfluxDB Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [kernel, stdlib, influxdb]}, {env, []}, diff --git a/changes/ee/feat-10679.en.md b/changes/ee/feat-10679.en.md new file mode 100644 index 000000000..ecd4bb8b8 --- /dev/null +++ b/changes/ee/feat-10679.en.md @@ -0,0 +1 @@ +Refactor the directory structure of the InfluxDB data bridge. From dcd4640a57602fb9f741ca13717fca6e070c849b Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 12 May 2023 14:28:26 +0800 Subject: [PATCH 075/144] fix: update the will message timestamp when it is ready to publish --- apps/emqx/src/emqx_channel.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 862b72c06..b8e0233fb 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -2146,7 +2146,8 @@ publish_will_msg( ok; false -> NMsg = emqx_mountpoint:mount(MountPoint, Msg), - _ = emqx_broker:publish(NMsg), + NMsg2 = NMsg#message{timestamp = erlang:system_time(millisecond)}, + _ = emqx_broker:publish(NMsg2), ok end. From 2404d167c8763d8079021e98a1388d89d6fea187 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 12 May 2023 14:39:18 +0800 Subject: [PATCH 076/144] chore: update changes --- changes/ce/fix-10682.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-10682.en.md diff --git a/changes/ce/fix-10682.en.md b/changes/ce/fix-10682.en.md new file mode 100644 index 000000000..df8d93116 --- /dev/null +++ b/changes/ce/fix-10682.en.md @@ -0,0 +1 @@ +Fix the timestamp for the will message is incorrectly assigned at the session creation time, now this timestamp is the disconnected time of the session. From 90549abec7e6ebf7a6af570ea159b6eb7e7a36bf Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 10 May 2023 14:49:34 +0800 Subject: [PATCH 077/144] fix: crash when sysmon.os.mem_check_interval = disabled --- apps/emqx/src/emqx_os_mon.erl | 17 +++--------- apps/emqx/test/emqx_os_mon_SUITE.erl | 39 ++++++++++++++++++---------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index 4810798eb..144d2bfe5 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -23,8 +23,6 @@ -export([start_link/0]). -export([ - get_mem_check_interval/0, - set_mem_check_interval/1, get_sysmem_high_watermark/0, set_sysmem_high_watermark/1, get_procmem_high_watermark/0, @@ -46,6 +44,9 @@ terminate/2, code_change/3 ]). +-ifdef(TEST). +-export([is_sysmem_check_supported/0]). +-endif. -include("emqx.hrl"). @@ -61,14 +62,6 @@ update(OS) -> %% API %%-------------------------------------------------------------------- -get_mem_check_interval() -> - memsup:get_check_interval(). - -set_mem_check_interval(Seconds) when Seconds < 60000 -> - memsup:set_check_interval(1); -set_mem_check_interval(Seconds) -> - memsup:set_check_interval(Seconds div 60000). - get_sysmem_high_watermark() -> gen_server:call(?OS_MON, ?FUNCTION_NAME, infinity). @@ -103,11 +96,9 @@ init_os_monitor() -> init_os_monitor(OS) -> #{ sysmem_high_watermark := SysHW, - procmem_high_watermark := PHW, - mem_check_interval := MCI + procmem_high_watermark := PHW } = OS, set_procmem_high_watermark(PHW), - set_mem_check_interval(MCI), ok = update_mem_alarm_status(SysHW), SysHW. diff --git a/apps/emqx/test/emqx_os_mon_SUITE.erl b/apps/emqx/test/emqx_os_mon_SUITE.erl index 0538d949a..e76928114 100644 --- a/apps/emqx/test/emqx_os_mon_SUITE.erl +++ b/apps/emqx/test/emqx_os_mon_SUITE.erl @@ -43,8 +43,8 @@ init_per_testcase(t_cpu_check_alarm, Config) -> {ok, _} = supervisor:restart_child(emqx_sys_sup, emqx_os_mon), Config; init_per_testcase(t_sys_mem_check_alarm, Config) -> - case os:type() of - {unix, linux} -> + case emqx_os_mon:is_sysmem_check_supported() of + true -> SysMon = emqx_config:get([sysmon, os], #{}), emqx_config:put([sysmon, os], SysMon#{ sysmem_high_watermark => 0.51, @@ -54,7 +54,7 @@ init_per_testcase(t_sys_mem_check_alarm, Config) -> ok = supervisor:terminate_child(emqx_sys_sup, emqx_os_mon), {ok, _} = supervisor:restart_child(emqx_sys_sup, emqx_os_mon), Config; - _ -> + false -> Config end; init_per_testcase(_, Config) -> @@ -63,12 +63,6 @@ init_per_testcase(_, Config) -> Config. t_api(_) -> - ?assertEqual(60000, emqx_os_mon:get_mem_check_interval()), - ?assertEqual(ok, emqx_os_mon:set_mem_check_interval(30000)), - ?assertEqual(60000, emqx_os_mon:get_mem_check_interval()), - ?assertEqual(ok, emqx_os_mon:set_mem_check_interval(122000)), - ?assertEqual(120000, emqx_os_mon:get_mem_check_interval()), - ?assertEqual(0.7, emqx_os_mon:get_sysmem_high_watermark()), ?assertEqual(ok, emqx_os_mon:set_sysmem_high_watermark(0.8)), ?assertEqual(0.8, emqx_os_mon:get_sysmem_high_watermark()), @@ -86,12 +80,29 @@ t_api(_) -> gen_server:stop(emqx_os_mon), ok. +t_sys_mem_check_disable(Config) -> + case emqx_os_mon:is_sysmem_check_supported() of + true -> do_sys_mem_check_disable(Config); + false -> skip + end. + +do_sys_mem_check_disable(_Config) -> + MemRef0 = maps:get(mem_time_ref, sys:get_state(emqx_os_mon)), + ?assertEqual(true, is_reference(MemRef0), MemRef0), + emqx_config:put([sysmon, os, mem_check_interval], 1000), + emqx_os_mon:update(emqx_config:get([sysmon, os])), + MemRef1 = maps:get(mem_time_ref, sys:get_state(emqx_os_mon)), + ?assertEqual(true, is_reference(MemRef1), {MemRef0, MemRef1}), + ?assertNotEqual(MemRef0, MemRef1), + emqx_config:put([sysmon, os, mem_check_interval], disabled), + emqx_os_mon:update(emqx_config:get([sysmon, os])), + ?assertEqual(undefined, maps:get(mem_time_ref, sys:get_state(emqx_os_mon))), + ok. + t_sys_mem_check_alarm(Config) -> - case os:type() of - {unix, linux} -> - do_sys_mem_check_alarm(Config); - _ -> - skip + case emqx_os_mon:is_sysmem_check_supported() of + true -> do_sys_mem_check_alarm(Config); + false -> skip end. do_sys_mem_check_alarm(_Config) -> From 1141499b1b4adf6c204f996f791c2b878c598bad Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 10 May 2023 14:56:56 +0800 Subject: [PATCH 078/144] chore: add changelog for 10659 --- changes/ce/fix-10659.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-10659.en.md diff --git a/changes/ce/fix-10659.en.md b/changes/ce/fix-10659.en.md new file mode 100644 index 000000000..97cf86e42 --- /dev/null +++ b/changes/ce/fix-10659.en.md @@ -0,0 +1 @@ +Fix the issue where emqx cannot start when `sysmon.os.mem_check_interval` is disabled. From 9ff901ac024dac55b4f28c5ffffc268f87acb6d8 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Sat, 6 May 2023 17:20:05 +0800 Subject: [PATCH 079/144] chore: upgarde rulesql to 0.1.6 to fix invaid utf8 input --- mix.exs | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index caee134c8..0284ed54a 100644 --- a/mix.exs +++ b/mix.exs @@ -65,7 +65,7 @@ defmodule EMQXUmbrella.MixProject do # maybe forbid to fetch quicer {:emqtt, github: "emqx/emqtt", tag: "1.8.5", override: true, system_env: maybe_no_quic_env()}, - {:rulesql, github: "emqx/rulesql", tag: "0.1.5"}, + {:rulesql, github: "emqx/rulesql", tag: "0.1.6"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, {:telemetry, "1.1.0"}, diff --git a/rebar.config b/rebar.config index 03b03345d..55a4583de 100644 --- a/rebar.config +++ b/rebar.config @@ -70,7 +70,7 @@ , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}} - , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}} + , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.6"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} From 88ab9c224d8ad87d932ec686de63fd133d366e58 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Fri, 12 May 2023 15:31:59 +0800 Subject: [PATCH 080/144] chore: upgrade dashboard to v1.2.4-1 for ce --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 24af58ebc..1ad4421aa 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ endif # Dashbord version # from https://github.com/emqx/emqx-dashboard5 -export EMQX_DASHBOARD_VERSION ?= v1.2.4 +export EMQX_DASHBOARD_VERSION ?= v1.2.4-1 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.6 # `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used From a38b270b03ad22ac03088e7fa34ebd0c075147a3 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 12 May 2023 17:27:32 +0800 Subject: [PATCH 081/144] fix(limiter): fix that update node-level limiter config will not working --- .../emqx_limiter/src/emqx_limiter_manager.erl | 11 ++++------ .../emqx_limiter/src/emqx_limiter_schema.erl | 21 ++++++++++++++++++- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl index 40061e0b9..afabc2580 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl @@ -131,11 +131,9 @@ delete_root(Type) -> delete_bucket(?ROOT_ID, Type). post_config_update([limiter], _Config, NewConf, _OldConf, _AppEnvs) -> - Types = lists:delete(client, maps:keys(NewConf)), - _ = [on_post_config_update(Type, NewConf) || Type <- Types], - ok; -post_config_update([limiter, Type], _Config, NewConf, _OldConf, _AppEnvs) -> - on_post_config_update(Type, NewConf). + Conf = emqx_limiter_schema:convert_node_opts(NewConf), + _ = [on_post_config_update(Type, Cfg) || {Type, Cfg} <- maps:to_list(Conf)], + ok. %%-------------------------------------------------------------------- %% @doc @@ -279,8 +277,7 @@ format_status(_Opt, Status) -> %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- -on_post_config_update(Type, NewConf) -> - Config = maps:get(Type, NewConf), +on_post_config_update(Type, Config) -> case emqx_limiter_server:whereis(Type) of undefined -> start_server(Type, Config); diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index 81358fdfe..667a38396 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -38,7 +38,8 @@ default_client_config/0, short_paths_fields/1, get_listener_opts/1, - get_node_opts/1 + get_node_opts/1, + convert_node_opts/1 ]). -define(KILOBYTE, 1024). @@ -309,6 +310,24 @@ get_node_opts(Type) -> end end. +convert_node_opts(Conf) -> + DefBucket = default_bucket_config(), + ShorPaths = short_paths(), + Fun = fun + %% The `client` in the node options was deprecated + (client, _Value, Acc) -> + Acc; + (Name, Value, Acc) -> + case lists:member(Name, ShorPaths) of + true -> + Type = short_path_name_to_type(Name), + Acc#{Type => DefBucket#{rate => Value}}; + _ -> + Acc#{Name => Value} + end + end, + maps:fold(Fun, #{}, Conf). + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- From 95a67f390fc82aa1c214335a322a8b879af43375 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 12 May 2023 18:39:51 +0800 Subject: [PATCH 082/144] fix(limiter): adjust type for compatibility --- apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index 667a38396..b9643dae4 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -61,7 +61,7 @@ -type limiter_id() :: atom(). -type bucket_name() :: atom(). -type rate() :: infinity | float(). --type burst_rate() :: 0 | float(). +-type burst_rate() :: number(). %% this is a compatible type for the deprecated field and type `capacity`. -type burst() :: burst_rate(). %% the capacity of the token bucket From 5ef2a603a12ac7f8335dcc27eb72bc99faaa0ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Fri, 12 May 2023 20:27:36 +0800 Subject: [PATCH 083/144] chore: bump to v5.0.25 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 2bb5877f1..1b4f8a6b6 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.25-rc.1"). +-define(EMQX_RELEASE_CE, "5.0.25"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.4-alpha.1"). From 02c8894dd376fa8a8d02af41dc1971e9eec1790f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 12 May 2023 13:24:32 -0300 Subject: [PATCH 084/144] fix(mix): fix `lhttpc` version specification --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 7b76bdc4b..c4e091e31 100644 --- a/mix.exs +++ b/mix.exs @@ -193,7 +193,7 @@ defmodule EMQXUmbrella.MixProject do {:erlcloud, github: "emqx/erlcloud", tag: "3.5.16-emqx-1", override: true}, # erlcloud's rebar.config requires rebar3 and does not support Mix, # so it tries to fetch deps from git. We need to override this. - {:lhttpc, tag: "1.6.2", override: true}, + {:lhttpc, "1.6.2", override: true}, {:eini, "1.2.9", override: true}, {:base16, "1.0.0", override: true}, # end of erlcloud's deps From 47c1fb734ab4618e9ab1a0054473532f8d3abd94 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 12 May 2023 21:12:14 +0200 Subject: [PATCH 085/144] ci: update performance test --- .github/workflows/performance_test.yaml | 52 ++++++++++++------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/.github/workflows/performance_test.yaml b/.github/workflows/performance_test.yaml index 1d474f7b2..6de9ddd21 100644 --- a/.github/workflows/performance_test.yaml +++ b/.github/workflows/performance_test.yaml @@ -51,11 +51,10 @@ jobs: needs: - prepare env: - TF_VAR_bench_id: ${{ needs.prepare.outputs.BENCH_ID }} TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }} - TF_VAR_test_duration: 300 TF_VAR_grafana_api_key: ${{ secrets.TF_EMQX_PERF_TEST_GRAFANA_API_KEY }} TF_AWS_REGION: eu-north-1 + TF_VAR_test_duration: 1800 steps: - name: Configure AWS Credentials @@ -77,38 +76,37 @@ jobs: uses: hashicorp/setup-terraform@v2 with: terraform_wrapper: false - - name: terraform init + - name: 1on1 scenario + id: scenario_1on1 working-directory: ./tf-emqx-performance-test + timeout-minutes: 60 + env: + TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/1on1" + TF_VAR_use_emqttb: 1 + TF_VAR_use_emqtt_bench: 0 + TF_VAR_emqttb_instance_count: 2 + TF_VAR_emqttb_instance_type: "c5.large" + TF_VAR_emqttb_scenario: "@pub --topic 't/%n' --pubinterval 10ms --qos 1 --publatency 50ms --size 16 --num-clients 25000 @sub --topic 't/%n' --num-clients 25000" + TF_VAR_emqx_instance_type: "c5.xlarge" + TF_VAR_emqx_instance_count: 3 run: | terraform init - - name: terraform apply - working-directory: ./tf-emqx-performance-test - run: | terraform apply -auto-approve - - name: Wait for test results - timeout-minutes: 30 - working-directory: ./tf-emqx-performance-test - id: test-results - run: | - sleep $TF_VAR_test_duration - until aws s3api head-object --bucket tf-emqx-performance-test --key "$TF_VAR_bench_id/DONE" > /dev/null 2>&1 - do - printf '.' - sleep 10 - done - echo - aws s3 cp "s3://tf-emqx-performance-test/$TF_VAR_bench_id/metrics.json" ./ - aws s3 cp "s3://tf-emqx-performance-test/$TF_VAR_bench_id/stats.json" ./ - echo MESSAGES_DELIVERED=$(cat metrics.json | jq '[.[]."messages.delivered"] | add') >> $GITHUB_OUTPUT - echo MESSAGES_DROPPED=$(cat metrics.json | jq '[.[]."messages.dropped"] | add') >> $GITHUB_OUTPUT + ./wait-emqttb.sh + ./fetch-metrics.sh + MESSAGES_RECEIVED=$(cat metrics.json | jq '[.[]."messages.received"] | add') + MESSAGES_SENT=$(cat metrics.json | jq '[.[]."messages.sent"] | add') + echo MESSAGES_DROPPED=$(cat metrics.json | jq '[.[]."messages.dropped"] | add') >> $GITHUB_OUTPUT + echo PUB_MSG_RATE=$(($MESSAGES_RECEIVED / $TF_VAR_test_duration)) >> $GITHUB_OUTPUT + echo SUB_MSG_RATE=$(($MESSAGES_SENT / $TF_VAR_test_duration)) >> $GITHUB_OUTPUT + terraform destroy -auto-approve - name: Send notification to Slack - if: success() uses: slackapi/slack-github-action@v1.23.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} with: payload: | - {"text": "EMQX performance test completed.\nMessages delivered: ${{ steps.test-results.outputs.MESSAGES_DELIVERED }}.\nMessages dropped: ${{ steps.test-results.outputs.MESSAGES_DROPPED }}.\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + {"text": "Performance test result for 1on1 scenario (50k pub, 50k sub): ${{ job.status }}\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}\n*Pub message rate*: ${{ steps.scenario_1on1.outputs.PUB_MSG_RATE }}\n*Sub message rate*: ${{ steps.scenario_1on1.outputs.SUB_MSG_RATE }}\nDropped messages: ${{ steps.scenario_1on1.outputs.MESSAGES_DROPPED }}"} - name: terraform destroy if: always() working-directory: ./tf-emqx-performance-test @@ -117,10 +115,10 @@ jobs: - uses: actions/upload-artifact@v3 if: success() with: - name: test-results - path: "./tf-emqx-performance-test/*.json" + name: metrics + path: "./tf-emqx-performance-test/metrics.json" - uses: actions/upload-artifact@v3 - if: always() + if: failure() with: name: terraform path: | From b3a989970d31e6b2c4d948f77b7da3826232f05d Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Sun, 14 May 2023 15:16:48 +0200 Subject: [PATCH 086/144] ci: move package upload to packagecloud.io to release workflow Fixes #10693 --- .github/workflows/build_packages.yaml | 32 ------------------- .github/workflows/release.yaml | 44 ++++++++++++++++++++------- 2 files changed, 33 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index eab9dc115..c88232dc9 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -306,35 +306,3 @@ jobs: fi aws s3 cp --recursive packages/$PROFILE s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$s3dir/${{ github.ref_name }}/*" - - name: Push to packagecloud.io - env: - PROFILE: ${{ matrix.profile }} - VERSION: ${{ needs.prepare.outputs.VERSION }} - PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - run: | - set -eu - REPO=$PROFILE - if [ $PROFILE = 'emqx-enterprise' ]; then - REPO='emqx-enterprise5' - fi - function push() { - docker run -t --rm -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN -v $(pwd)/$2:/w/$2 -w /w ghcr.io/emqx/package_cloud push emqx/$REPO/$1 $2 - } - push "debian/buster" "packages/$PROFILE/$PROFILE-$VERSION-debian10-amd64.deb" - push "debian/buster" "packages/$PROFILE/$PROFILE-$VERSION-debian10-arm64.deb" - push "debian/bullseye" "packages/$PROFILE/$PROFILE-$VERSION-debian11-amd64.deb" - push "debian/bullseye" "packages/$PROFILE/$PROFILE-$VERSION-debian11-arm64.deb" - push "ubuntu/bionic" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu18.04-amd64.deb" - push "ubuntu/bionic" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu18.04-arm64.deb" - push "ubuntu/focal" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu20.04-amd64.deb" - push "ubuntu/focal" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu20.04-arm64.deb" - push "ubuntu/jammy" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu22.04-amd64.deb" - push "ubuntu/jammy" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu22.04-arm64.deb" - push "el/6" "packages/$PROFILE/$PROFILE-$VERSION-amzn2-amd64.rpm" - push "el/6" "packages/$PROFILE/$PROFILE-$VERSION-amzn2-arm64.rpm" - push "el/7" "packages/$PROFILE/$PROFILE-$VERSION-el7-amd64.rpm" - push "el/7" "packages/$PROFILE/$PROFILE-$VERSION-el7-arm64.rpm" - push "el/8" "packages/$PROFILE/$PROFILE-$VERSION-el8-amd64.rpm" - push "el/8" "packages/$PROFILE/$PROFILE-$VERSION-el8-arm64.rpm" - push "el/9" "packages/$PROFILE/$PROFILE-$VERSION-el9-amd64.rpm" - push "el/9" "packages/$PROFILE/$PROFILE-$VERSION-el9-arm64.rpm" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 32a45bd51..30de6f3b1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -15,7 +15,7 @@ on: jobs: upload: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false steps: @@ -53,16 +53,6 @@ jobs: BUCKET=${{ secrets.AWS_S3_BUCKET }} OUTPUT_DIR=${{ steps.profile.outputs.s3dir }} aws s3 cp --recursive s3://$BUCKET/$OUTPUT_DIR/${{ github.ref_name }} packages - cd packages - DEFAULT_BEAM_PLATFORM='otp24.3.4.2-3' - # all packages including full-name and default-name are uploaded to s3 - # but we only upload default-name packages (and elixir) as github artifacts - # so we rename (overwrite) non-default packages before uploading - while read -r fname; do - default_fname=$(echo "$fname" | sed "s/-${DEFAULT_BEAM_PLATFORM}//g") - echo "$fname -> $default_fname" - mv -f "$fname" "$default_fname" - done < <(find . -maxdepth 1 -type f | grep -E "emqx(-enterprise)?-5\.[0-9]+\.[0-9]+.*-${DEFAULT_BEAM_PLATFORM}" | grep -v elixir) - uses: alexellis/upload-assets@0.4.0 env: GITHUB_TOKEN: ${{ github.token }} @@ -79,3 +69,35 @@ jobs: -X POST \ -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} + - name: Push to packagecloud.io + env: + PROFILE: ${{ steps.profile.outputs.profile }} + VERSION: ${{ steps.profile.outputs.version }} + PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} + run: | + set -eu + REPO=$PROFILE + if [ $PROFILE = 'emqx-enterprise' ]; then + REPO='emqx-enterprise5' + fi + function push() { + docker run -t --rm -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN -v $(pwd)/$2:/w/$2 -w /w ghcr.io/emqx/package_cloud push emqx/$REPO/$1 $2 + } + push "debian/buster" "packages/$PROFILE-$VERSION-debian10-amd64.deb" + push "debian/buster" "packages/$PROFILE-$VERSION-debian10-arm64.deb" + push "debian/bullseye" "packages/$PROFILE-$VERSION-debian11-amd64.deb" + push "debian/bullseye" "packages/$PROFILE-$VERSION-debian11-arm64.deb" + push "ubuntu/bionic" "packages/$PROFILE-$VERSION-ubuntu18.04-amd64.deb" + push "ubuntu/bionic" "packages/$PROFILE-$VERSION-ubuntu18.04-arm64.deb" + push "ubuntu/focal" "packages/$PROFILE-$VERSION-ubuntu20.04-amd64.deb" + push "ubuntu/focal" "packages/$PROFILE-$VERSION-ubuntu20.04-arm64.deb" + push "ubuntu/jammy" "packages/$PROFILE-$VERSION-ubuntu22.04-amd64.deb" + push "ubuntu/jammy" "packages/$PROFILE-$VERSION-ubuntu22.04-arm64.deb" + push "el/6" "packages/$PROFILE-$VERSION-amzn2-amd64.rpm" + push "el/6" "packages/$PROFILE-$VERSION-amzn2-arm64.rpm" + push "el/7" "packages/$PROFILE-$VERSION-el7-amd64.rpm" + push "el/7" "packages/$PROFILE-$VERSION-el7-arm64.rpm" + push "el/8" "packages/$PROFILE-$VERSION-el8-amd64.rpm" + push "el/8" "packages/$PROFILE-$VERSION-el8-arm64.rpm" + push "el/9" "packages/$PROFILE-$VERSION-el9-amd64.rpm" + push "el/9" "packages/$PROFILE-$VERSION-el9-arm64.rpm" From 383fec9dfc6e9798dac3ee53f199597abe446d7f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 15 May 2023 11:18:09 +0800 Subject: [PATCH 087/144] docs: fix invalid links --- apps/emqx_bridge_cassandra/README.md | 2 -- apps/emqx_bridge_dynamo/README.md | 2 +- apps/emqx_bridge_gcp_pubsub/README.md | 2 +- apps/emqx_bridge_influxdb/README.md | 2 +- apps/emqx_bridge_iotdb/README.md | 2 +- apps/emqx_bridge_kafka/README.md | 2 +- apps/emqx_bridge_mongodb/README.md | 2 +- apps/emqx_bridge_mysql/README.md | 2 +- apps/emqx_bridge_pgsql/README.md | 4 ++-- apps/emqx_bridge_redis/README.md | 2 +- apps/emqx_bridge_rocketmq/README.md | 2 +- apps/emqx_bridge_sqlserver/README.md | 2 +- apps/emqx_bridge_tdengine/README.md | 2 +- apps/emqx_gateway/README.md | 4 ++-- apps/emqx_gateway_exproto/README.md | 4 ---- apps/emqx_rule_engine/README.md | 2 +- 16 files changed, 16 insertions(+), 22 deletions(-) diff --git a/apps/emqx_bridge_cassandra/README.md b/apps/emqx_bridge_cassandra/README.md index c5a2609a5..d26bd2fbb 100644 --- a/apps/emqx_bridge_cassandra/README.md +++ b/apps/emqx_bridge_cassandra/README.md @@ -11,7 +11,6 @@ The application is used to connect EMQX and Cassandra. User can create a rule and easily ingest IoT data into Cassandra by leveraging [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). - # HTTP APIs diff --git a/apps/emqx_bridge_dynamo/README.md b/apps/emqx_bridge_dynamo/README.md index 48dcb781d..245a68941 100644 --- a/apps/emqx_bridge_dynamo/README.md +++ b/apps/emqx_bridge_dynamo/README.md @@ -1,6 +1,6 @@ # EMQX DynamoDB Bridge -[Dynamodb](https://aws.amazon.com/dynamodb/) is a high-performance NoSQL database +[DynamoDB](https://aws.amazon.com/dynamodb/) is a high-performance NoSQL database service provided by Amazon that's designed for scalability and low-latency access to structured data. diff --git a/apps/emqx_bridge_gcp_pubsub/README.md b/apps/emqx_bridge_gcp_pubsub/README.md index e33c5ab15..6d136a5d3 100644 --- a/apps/emqx_bridge_gcp_pubsub/README.md +++ b/apps/emqx_bridge_gcp_pubsub/README.md @@ -10,7 +10,7 @@ User can create a rule and easily ingest IoT data into GCP Pub/Sub by leveraging # Documentation -- Refer to [Ingest data into GCP Pub/Sub](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-gcp-pubsub.html) +- Refer to [Ingest Data into GCP Pub/Sub](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-gcp-pubsub.html) for how to use EMQX dashboard to ingest IoT data into GCP Pub/Sub. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_bridge_influxdb/README.md b/apps/emqx_bridge_influxdb/README.md index fe0f14600..69df002c4 100644 --- a/apps/emqx_bridge_influxdb/README.md +++ b/apps/emqx_bridge_influxdb/README.md @@ -15,7 +15,7 @@ easily ingest IoT data into InfluxDB by leveraging # Documentation -- Refer to [Ingest data into InfluxDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-influxdb.html) +- Refer to [Ingest Data into InfluxDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-influxdb.html) for how to use EMQX dashboard to ingest IoT data into InfluxDB. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_bridge_iotdb/README.md b/apps/emqx_bridge_iotdb/README.md index 48f5d74c2..306076507 100644 --- a/apps/emqx_bridge_iotdb/README.md +++ b/apps/emqx_bridge_iotdb/README.md @@ -18,7 +18,7 @@ Please see [our official documentation](https://www.emqx.io/docs/en/v5.0/data-integration/data-bridge-iotdb.html) for more detailed info. -# Contributing - [Mandatory] +# Contributing Please see our [contributing.md](../../CONTRIBUTING.md). # License diff --git a/apps/emqx_bridge_kafka/README.md b/apps/emqx_bridge_kafka/README.md index 07cae256b..56c340a70 100644 --- a/apps/emqx_bridge_kafka/README.md +++ b/apps/emqx_bridge_kafka/README.md @@ -16,7 +16,7 @@ For more information about Apache Kafka, please see its [official site](https:// # Configurations -Please see [Ingest data into Kafka](https://www.emqx.io/docs/en/v5.0/data-integration/data-bridge-kafka.html) for more detailed info. +Please see [Ingest Data into Kafka](https://www.emqx.io/docs/en/v5.0/data-integration/data-bridge-kafka.html) for more detailed info. # Contributing diff --git a/apps/emqx_bridge_mongodb/README.md b/apps/emqx_bridge_mongodb/README.md index 088c8467f..63a541dc5 100644 --- a/apps/emqx_bridge_mongodb/README.md +++ b/apps/emqx_bridge_mongodb/README.md @@ -13,7 +13,7 @@ User can create a rule and easily ingest IoT data into MongoDB by leveraging # Documentation -- Refer to [Ingest data into MongoDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-mongodb.html) +- Refer to [Ingest Data into MongoDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-mongodb.html) for how to use EMQX dashboard to ingest IoT data into MongoDB. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_bridge_mysql/README.md b/apps/emqx_bridge_mysql/README.md index d7c9b5647..8b4295e96 100644 --- a/apps/emqx_bridge_mysql/README.md +++ b/apps/emqx_bridge_mysql/README.md @@ -10,7 +10,7 @@ User can create a rule and easily ingest IoT data into MySQL by leveraging # Documentation -- Refer to [Ingest data into MySQL](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-mysql.html) +- Refer to [Ingest Data into MySQL](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-mysql.html) for how to use EMQX dashboard to ingest IoT data into MySQL. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_bridge_pgsql/README.md b/apps/emqx_bridge_pgsql/README.md index fc0bd3c6f..20e8ef86d 100644 --- a/apps/emqx_bridge_pgsql/README.md +++ b/apps/emqx_bridge_pgsql/README.md @@ -1,6 +1,6 @@ # EMQX PostgreSQL Bridge -[PostgreSQL](https://github.com/PostgreSQL/PostgreSQL) is an open-source relational +[PostgreSQL](https://www.postgresql.org/) is an open-source relational database management system (RDBMS) that uses and extends the SQL language. It is known for its reliability, data integrity, and advanced features such as support for JSON, XML, and other data formats. @@ -12,7 +12,7 @@ User can create a rule and easily ingest IoT data into PostgreSQL by leveraging # Documentation -- Refer to [Ingest data into PostgreSQL](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-pgsql.html) +- Refer to [Ingest Data into PostgreSQL](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-pgsql.html) for how to use EMQX dashboard to ingest IoT data into PostgreSQL. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_bridge_redis/README.md b/apps/emqx_bridge_redis/README.md index 73ec41f07..5aca9fddb 100644 --- a/apps/emqx_bridge_redis/README.md +++ b/apps/emqx_bridge_redis/README.md @@ -11,7 +11,7 @@ User can create a rule and easily ingest IoT data into Redis by leveraging # Documentation -- Refer to [Ingest data into Redis](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-redis.html) +- Refer to [Ingest Data into Redis](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-redis.html) for how to use EMQX dashboard to ingest IoT data into Redis. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_bridge_rocketmq/README.md b/apps/emqx_bridge_rocketmq/README.md index 252e6beac..e6b1ceff3 100644 --- a/apps/emqx_bridge_rocketmq/README.md +++ b/apps/emqx_bridge_rocketmq/README.md @@ -11,7 +11,7 @@ User can create a rule and easily ingest IoT data into RocketMQ by leveraging # Documentation -- Refer to [Ingest data into RocketMQ](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-rocketmq.html) +- Refer to [Ingest Data into RocketMQ](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-rocketmq.html) for how to use EMQX dashboard to ingest IoT data into RocketMQ. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_bridge_sqlserver/README.md b/apps/emqx_bridge_sqlserver/README.md index ccb1267d8..629560036 100644 --- a/apps/emqx_bridge_sqlserver/README.md +++ b/apps/emqx_bridge_sqlserver/README.md @@ -16,7 +16,7 @@ For more information about Microsoft SQL Server, please see the [official site]( # Configurations -Please see [Ingest data into SQL Server](https://www.emqx.io/docs/en/v5.0/data-integration/data-bridge-sqlserver.html) for more detailed information. +Please see [Ingest Data into SQL Server](https://www.emqx.io/docs/en/v5.0/data-integration/data-bridge-sqlserver.html) for more detailed information. # HTTP APIs diff --git a/apps/emqx_bridge_tdengine/README.md b/apps/emqx_bridge_tdengine/README.md index 25faf4c14..e32e16f09 100644 --- a/apps/emqx_bridge_tdengine/README.md +++ b/apps/emqx_bridge_tdengine/README.md @@ -13,7 +13,7 @@ User can create a rule and easily ingest IoT data into TDEngine by leveraging # Documentation -- Refer to [Ingest data into TDEngine](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-tdengine.html) +- Refer to [Ingest Data into TDEngine](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-tdengine.html) for how to use EMQX dashboard to ingest IoT data into TDEngine. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) diff --git a/apps/emqx_gateway/README.md b/apps/emqx_gateway/README.md index ebab3a7a9..c71737639 100644 --- a/apps/emqx_gateway/README.md +++ b/apps/emqx_gateway/README.md @@ -19,8 +19,8 @@ More introduction: [Extended Protocol Gateway](https://www.emqx.io/docs/en/v5.0/ ## Usage This application is just a Framework, we provide some standard implementations, -such as [Stomp](../emqx_stomp/README.md), [MQTT-SN](../emqx_mqttsn/README.md), -[CoAP](../emqx_coap/README.md) and [LwM2M](../emqx_lwm2m/README.md) gateway. +such as [Stomp](../emqx_gateway_stomp/README.md), [MQTT-SN](../emqx_gateway_mqttsn/README.md), +[CoAP](../emqx_gateway_coap/README.md) and [LwM2M](../emqx_gateway_lwm2m/README.md) gateway. These applications are all packaged by default in the EMQX distribution. If you need to start a certain gateway, you only need to enable it via diff --git a/apps/emqx_gateway_exproto/README.md b/apps/emqx_gateway_exproto/README.md index a2268f0dc..ec8b93984 100644 --- a/apps/emqx_gateway_exproto/README.md +++ b/apps/emqx_gateway_exproto/README.md @@ -7,10 +7,6 @@ The `emqx_exproto` extremely enhance the extensibility for EMQX. It allow using - [x] Based on gRPC, it brings a very wide range of applicability - [x] Allows you to use the return value to extend emqx behavior. -## Architecture - -![EMQX ExProto Arch](./docs/images/exproto-arch.jpg) - ## Usage ### gRPC service diff --git a/apps/emqx_rule_engine/README.md b/apps/emqx_rule_engine/README.md index 2c2e43db3..3fdd9b6a7 100644 --- a/apps/emqx_rule_engine/README.md +++ b/apps/emqx_rule_engine/README.md @@ -1,5 +1,5 @@ -# Emqx Rule Engine +# EMQX Rule Engine The rule engine's goal is to provide a simple and flexible way to transform and reroute the messages coming to the EMQX broker. For example, one message From 38fcb7a097a59e2bd89ca0089f52ee57596950c5 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 15 May 2023 11:20:23 +0800 Subject: [PATCH 088/144] docs: hide the not-ready document links --- apps/emqx_bridge_iotdb/README.md | 2 ++ apps/emqx_bridge_pulsar/README.md | 2 ++ apps/emqx_bridge_rabbitmq/README.md | 2 ++ 3 files changed, 6 insertions(+) diff --git a/apps/emqx_bridge_iotdb/README.md b/apps/emqx_bridge_iotdb/README.md index 306076507..29b634dbc 100644 --- a/apps/emqx_bridge_iotdb/README.md +++ b/apps/emqx_bridge_iotdb/README.md @@ -12,11 +12,13 @@ It implements the connection management and interaction without need for a For more information on Apache IoTDB, please see its [official site](https://iotdb.apache.org/). + # Contributing Please see our [contributing.md](../../CONTRIBUTING.md). diff --git a/apps/emqx_bridge_pulsar/README.md b/apps/emqx_bridge_pulsar/README.md index fbd8bf81d..cbdaeff2f 100644 --- a/apps/emqx_bridge_pulsar/README.md +++ b/apps/emqx_bridge_pulsar/README.md @@ -15,11 +15,13 @@ used by authentication and authorization applications. For more information on Apache Pulsar, please see its [official site](https://pulsar.apache.org/). + # Contributing diff --git a/apps/emqx_bridge_rabbitmq/README.md b/apps/emqx_bridge_rabbitmq/README.md index 420a9e048..a85f5507d 100644 --- a/apps/emqx_bridge_rabbitmq/README.md +++ b/apps/emqx_bridge_rabbitmq/README.md @@ -21,8 +21,10 @@ and easily ingest IoT data into RabbitMQ by leveraging # Documentation + - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) for an introduction to the EMQX rules engine. From a264827b78764638423506dceb3920a442d87252 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 10 May 2023 14:22:05 +0200 Subject: [PATCH 089/144] ci: emqx-builder 5.0-35 --- .github/workflows/build_and_push_docker_images.yaml | 4 ++-- .github/workflows/build_packages.yaml | 8 ++++---- .github/workflows/build_packages_cron.yaml | 2 +- .github/workflows/build_slim_packages.yaml | 2 +- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/code_style_check.yaml | 2 +- .github/workflows/elixir_apps_check.yaml | 2 +- .github/workflows/elixir_deps_check.yaml | 2 +- .github/workflows/elixir_release.yml | 2 +- .github/workflows/performance_test.yaml | 2 +- .github/workflows/run_emqx_app_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 6 +++--- .github/workflows/run_relup_tests.yaml | 2 +- .github/workflows/run_test_cases.yaml | 6 +++--- 14 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 7c0a5dc87..64fc6d5b0 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -25,7 +25,7 @@ jobs: prepare: runs-on: ubuntu-22.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-24.3.4.2-3-ubuntu22.04" outputs: PROFILE: ${{ steps.get_profile.outputs.PROFILE }} @@ -121,7 +121,7 @@ jobs: # NOTE: 'otp' and 'elixir' are to configure emqx-builder image # only support latest otp and elixir, not a matrix builder: - - 5.0-34 # update to latest + - 5.0-35 # update to latest otp: - 24.3.4.2-3 # switch to 25 once ready to release 5.1 elixir: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index c88232dc9..be6c73068 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -21,7 +21,7 @@ on: jobs: prepare: runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-24.3.4.2-3-ubuntu22.04 outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} @@ -184,7 +184,7 @@ jobs: - aws-arm64 - ubuntu-22.04 builder: - - 5.0-34 + - 5.0-35 elixir: - 1.13.4 exclude: @@ -198,7 +198,7 @@ jobs: arch: amd64 os: ubuntu22.04 build_machine: ubuntu-22.04 - builder: 5.0-34 + builder: 5.0-35 elixir: 1.13.4 release_with: elixir - profile: emqx @@ -206,7 +206,7 @@ jobs: arch: amd64 os: amzn2 build_machine: ubuntu-22.04 - builder: 5.0-34 + builder: 5.0-35 elixir: 1.13.4 release_with: elixir diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index b7ef44a79..8498b30ba 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -32,7 +32,7 @@ jobs: - debian10 - amzn2 builder: - - 5.0-34 + - 5.0-35 elixir: - 1.13.4 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 06bcb98a2..836eaf079 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -35,7 +35,7 @@ jobs: - ["emqx-enterprise", "24.3.4.2-3", "amzn2", "erlang"] - ["emqx-enterprise", "25.1.2-3", "ubuntu20.04", "erlang"] builder: - - 5.0-34 + - 5.0-35 elixir: - '1.13.4' diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 52ebf9efc..83e328d21 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -6,7 +6,7 @@ on: jobs: check_deps_integrity: runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-25.1.2-3-ubuntu22.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml index 1508cdd6e..8baa7e721 100644 --- a/.github/workflows/code_style_check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: code_style_check: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-25.1.2-3-ubuntu22.04" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index 7e942f3f3..123b79566 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -9,7 +9,7 @@ jobs: elixir_apps_check: runs-on: ubuntu-22.04 # just use the latest builder - container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-25.1.2-3-ubuntu22.04" strategy: fail-fast: false diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index e967c186b..99afd109e 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -8,7 +8,7 @@ on: jobs: elixir_deps_check: runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-25.1.2-3-ubuntu22.04 steps: - name: Checkout diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index 9a916d332..14b6f941f 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -17,7 +17,7 @@ jobs: profile: - emqx - emqx-enterprise - container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-25.1.2-3-ubuntu22.04 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/performance_test.yaml b/.github/workflows/performance_test.yaml index 6de9ddd21..5283de51a 100644 --- a/.github/workflows/performance_test.yaml +++ b/.github/workflows/performance_test.yaml @@ -15,7 +15,7 @@ jobs: prepare: runs-on: ubuntu-latest if: github.repository_owner == 'emqx' - container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-25.1.2-3-ubuntu20.04 outputs: BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }} PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }} diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 551d0d9e6..20429392d 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: builder: - - 5.0-34 + - 5.0-35 otp: - 24.3.4.2-3 - 25.1.2-3 diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 185c76be1..b1246f745 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -17,7 +17,7 @@ jobs: prepare: runs-on: ubuntu-22.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-debian11 + container: ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-24.3.4.2-3-debian11 steps: - uses: actions/checkout@v3 @@ -50,7 +50,7 @@ jobs: os: - ["debian11", "debian:11-slim"] builder: - - 5.0-34 + - 5.0-35 otp: - 24.3.4.2-3 elixir: @@ -123,7 +123,7 @@ jobs: os: - ["debian11", "debian:11-slim"] builder: - - 5.0-34 + - 5.0-35 otp: - 24.3.4.2-3 elixir: diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 2ad7c3345..8f5d9f6ae 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -15,7 +15,7 @@ concurrency: jobs: relup_test_plan: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-24.3.4.2-3-ubuntu22.04" outputs: CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index b82b545df..bcd84d90c 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -34,12 +34,12 @@ jobs: MATRIX="$(echo "${APPS}" | jq -c ' [ (.[] | select(.profile == "emqx") | . + { - builder: "5.0-34", + builder: "5.0-35", otp: "25.1.2-3", elixir: "1.13.4" }), (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.0-34", + builder: "5.0-35", otp: ["24.3.4.2-3", "25.1.2-3"][], elixir: "1.13.4" }) @@ -255,7 +255,7 @@ jobs: - ct - ct_docker runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-35:1.13.4-24.3.4.2-3-ubuntu22.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 From 8d8cf23fad31b34b40e8150eb57bedab37a7b871 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 15 May 2023 08:17:30 +0200 Subject: [PATCH 090/144] chore: add changelog --- changes/ce/fix-10701.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-10701.en.md diff --git a/changes/ce/fix-10701.en.md b/changes/ce/fix-10701.en.md new file mode 100644 index 000000000..c153e01e1 --- /dev/null +++ b/changes/ce/fix-10701.en.md @@ -0,0 +1 @@ +RPM package for Amazon Linux 2 did not support TLS v1.3 as it was assembled with Erlang/OTP built with openssl 1.0. From 3c64735b876fa1581320a1db39a0bb00c2db360f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 11 May 2023 10:52:58 +0200 Subject: [PATCH 091/144] feat(rule_engine): support external SQL function provider --- apps/emqx_rule_engine/include/rule_engine.hrl | 15 +++++ .../src/emqx_rule_runtime.erl | 62 +++++++++++++++---- .../src/emqx_rule_sqlparser.erl | 22 +++++-- .../test/emqx_rule_engine_SUITE.erl | 59 ++++++++++++++++++ .../test/emqx_rule_funcs_demo.erl | 32 ++++++++++ 5 files changed, 174 insertions(+), 16 deletions(-) create mode 100644 apps/emqx_rule_engine/test/emqx_rule_funcs_demo.erl diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl index c69a24244..51ad6ab85 100644 --- a/apps/emqx_rule_engine/include/rule_engine.hrl +++ b/apps/emqx_rule_engine/include/rule_engine.hrl @@ -92,6 +92,8 @@ ?RAISE(EXP, _ = do_nothing, ERROR) ). +-define(RAISE_BAD_SQL(Detail), throw(Detail)). + -define(RAISE(EXP, EXP_ON_FAIL, ERROR), fun() -> try @@ -106,3 +108,16 @@ %% Tables -define(RULE_TAB, emqx_rule_engine). + +%% Allowed sql function provider modules +-define(DEFAULT_SQL_FUNC_PROVIDER, emqx_rule_funcs). +-define(IS_VALID_SQL_FUNC_PROVIDER_MODULE_NAME(Name), + (case Name of + <<"emqx_rule_funcs", _/binary>> -> + true; + <<"EmqxRuleFuncs", _/binary>> -> + true; + _ -> + false + end) +). diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index d7412d03c..9bc10f4d2 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -452,19 +452,23 @@ eval_switch_clauses(CaseOn, [{Cond, Clause} | CaseClauses], ElseClauses, Columns eval_switch_clauses(CaseOn, CaseClauses, ElseClauses, Columns) end. -apply_func(Name, Args, Columns) when is_atom(Name) -> - do_apply_func(Name, Args, Columns); apply_func(Name, Args, Columns) when is_binary(Name) -> - FunName = - try - binary_to_existing_atom(Name, utf8) - catch - error:badarg -> error({sql_function_not_supported, Name}) - end, - do_apply_func(FunName, Args, Columns). + FuncName = parse_function_name(?DEFAULT_SQL_FUNC_PROVIDER, Name), + apply_func(FuncName, Args, Columns); +apply_func([{key, ModuleName0}, {key, FuncName0}], Args, Columns) -> + ModuleName = parse_module_name(ModuleName0), + FuncName = parse_function_name(ModuleName, FuncName0), + do_apply_func(ModuleName, FuncName, Args, Columns); +apply_func(Name, Args, Columns) when is_atom(Name) -> + do_apply_func(?DEFAULT_SQL_FUNC_PROVIDER, Name, Args, Columns); +apply_func(Other, _, _) -> + ?RAISE_BAD_SQL(#{ + reason => bad_sql_function_reference, + reference => Other + }). -do_apply_func(Name, Args, Columns) -> - case erlang:apply(emqx_rule_funcs, Name, Args) of +do_apply_func(Module, Name, Args, Columns) -> + case erlang:apply(Module, Name, Args) of Func when is_function(Func) -> erlang:apply(Func, [Columns]); Result -> @@ -531,3 +535,39 @@ is_ok_result(R) when is_tuple(R) -> ok == erlang:element(1, R); is_ok_result(_) -> false. + +parse_module_name(Name) when is_binary(Name) -> + case ?IS_VALID_SQL_FUNC_PROVIDER_MODULE_NAME(Name) of + true -> + ok; + false -> + ?RAISE_BAD_SQL(#{ + reason => sql_function_provider_module_not_allowed, + module => Name + }) + end, + try + parse_module_name(binary_to_existing_atom(Name, utf8)) + catch + error:badarg -> + ?RAISE_BAD_SQL(#{ + reason => sql_function_provider_module_not_loaded, + module => Name + }) + end; +parse_module_name(Name) when is_atom(Name) -> + Name. + +parse_function_name(Module, Name) when is_binary(Name) -> + try + parse_function_name(Module, binary_to_existing_atom(Name, utf8)) + catch + error:badarg -> + ?RAISE_BAD_SQL(#{ + reason => sql_function_not_supported, + module => Module, + function => Name + }) + end; +parse_function_name(_Module, Name) when is_atom(Name) -> + Name. diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl b/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl index 9b6ed7eae..b6661684f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl @@ -44,11 +44,23 @@ -type alias() :: binary() | list(binary()). --type field() :: - const() - | variable() - | {as, field(), alias()} - | {'fun', atom(), list(field())}. +%% TODO: So far the SQL function module names and function names are as binary(), +%% binary_to_atom is called to convert to module and function name. +%% For better performance, the function references +%% can be converted to a fun Module:Function/N When compiling the SQL. +-type ext_module_name() :: atom() | binary(). +-type func_name() :: atom() | binary(). +-type func_args() :: [field()]. +%% Functions defiend in emqx_rule_funcs +-type builtin_func_ref() :: {var, func_name()}. +%% Functions defined in other modules, reference syntax: Module.Function(Arg1, Arg2, ...) +%% NOTE: it's '.' (Elixir style), but not ':' (Erlang style). +%% Parsed as a two element path-list: [{key, Module}, {key, Func}]. +-type external_func_ref() :: {path, [{key, ext_module_name() | func_name()}]}. +-type func_ref() :: builtin_func_ref() | external_func_ref(). +-type sql_func() :: {'fun', func_ref(), func_args()}. + +-type field() :: const() | variable() | {as, field(), alias()} | sql_func(). -export_type([select/0]). diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index eb253e516..f2218fa87 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -62,6 +62,9 @@ groups() -> t_match_atom_and_binary, t_sqlselect_0, t_sqlselect_00, + t_sqlselect_with_3rd_party_impl, + t_sqlselect_with_3rd_party_impl2, + t_sqlselect_with_3rd_party_funcs_unknown, t_sqlselect_001, t_sqlselect_inject_props, t_sqlselect_01, @@ -120,6 +123,8 @@ groups() -> %%------------------------------------------------------------------------------ init_per_suite(Config) -> + %% ensure module loaded + emqx_rule_funcs_demo:module_info(), application:load(emqx_conf), ok = emqx_common_test_helpers:start_apps( [emqx_conf, emqx_rule_engine, emqx_authz], @@ -1012,6 +1017,60 @@ t_sqlselect_00(_Config) -> ) ). +t_sqlselect_with_3rd_party_impl(_Config) -> + Sql = + "select * from \"t/#\" where emqx_rule_funcs_demo.is_my_topic(topic)", + T = fun(Topic) -> + emqx_rule_sqltester:test( + #{ + sql => Sql, + context => + #{ + payload => #{<<"what">> => 0}, + topic => Topic + } + } + ) + end, + ?assertMatch({ok, _}, T(<<"t/2/3/4/5">>)), + ?assertMatch({error, nomatch}, T(<<"t/1">>)). + +t_sqlselect_with_3rd_party_impl2(_Config) -> + Sql = fun(N) -> + "select emqx_rule_funcs_demo.duplicate_payload(payload," ++ integer_to_list(N) ++ + ") as payload_list from \"t/#\"" + end, + T = fun(Payload, N) -> + emqx_rule_sqltester:test( + #{ + sql => Sql(N), + context => + #{ + payload => Payload, + topic => <<"t/a">> + } + } + ) + end, + ?assertMatch({ok, #{<<"payload_list">> := [_, _]}}, T(<<"payload1">>, 2)), + ?assertMatch({ok, #{<<"payload_list">> := [_, _, _]}}, T(<<"payload1">>, 3)), + %% crash + ?assertMatch({error, {select_and_transform_error, _}}, T(<<"payload1">>, 4)). + +t_sqlselect_with_3rd_party_funcs_unknown(_Config) -> + Sql = "select emqx_rule_funcs_demo_no_such_module.foo(payload) from \"t/#\"", + ?assertMatch( + {error, + {select_and_transform_error, + {throw, #{reason := sql_function_provider_module_not_loaded}, _}}}, + emqx_rule_sqltester:test( + #{ + sql => Sql, + context => #{payload => <<"a">>, topic => <<"t/a">>} + } + ) + ). + t_sqlselect_001(_Config) -> %% Verify that the jq function can be called from SQL Sql = diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_demo.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_demo.erl new file mode 100644 index 000000000..b0d42b10e --- /dev/null +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_demo.erl @@ -0,0 +1,32 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_rule_funcs_demo). + +-export([ + is_my_topic/1, + duplicate_payload/2 +]). + +%% check if the topic is of 5 levels. +is_my_topic(Topic) -> + emqx_topic:levels(Topic) =:= 5. + +%% duplicate the payload, but only supports 2 or 3 copies. +duplicate_payload(Payload, 2) -> + [Payload, Payload]; +duplicate_payload(Payload, 3) -> + [Payload, Payload, Payload]. From 34b09e071fbdc4e91e5759eeb906387b0f01807c Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 15 May 2023 18:08:33 +0800 Subject: [PATCH 092/144] fix(limiter): fix a format error in the changelog --- changes/ce/perf-10625.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ce/perf-10625.en.md b/changes/ce/perf-10625.en.md index 42e712648..a3271d739 100644 --- a/changes/ce/perf-10625.en.md +++ b/changes/ce/perf-10625.en.md @@ -1,4 +1,4 @@ Simplify limiter configuration. - Reduce the complexity of the limiter's configuration. -e.g. now users can use `limiter.messages_rate = 1000/s` to quickly set the node-level limit for the message publish. +e.g. now users can use `limiter.messages_rate = "1000/s"` to quickly set the node-level limit for the message publish. - Update the `configs/limiter` API to suit this refactor. From 5960cc530a2cd147b64b1659c16aba73ec275c29 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 15 May 2023 09:05:21 -0300 Subject: [PATCH 093/144] fix(pulsar): use a binary duration as default `health_check_interval` Fixes https://emqx.atlassian.net/browse/EMQX-9885 The frontend needs the default value to match the duration (binary) type to display correctly. --- apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl index 18faf0e3b..5a87d8a0c 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl @@ -140,7 +140,7 @@ fields(producer_resource_opts) -> lists:filtermap( fun ({health_check_interval = Field, MetaFn}) -> - {true, {Field, override_default(MetaFn, 1_000)}}; + {true, {Field, override_default(MetaFn, <<"1s">>)}}; ({Field, _Meta}) -> lists:member(Field, SupportedOpts) end, From 46f05056deb3f89836c6a6afc5c24a8adfb4dedc Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 15 May 2023 16:29:52 +0200 Subject: [PATCH 094/144] docs: clarify description of bridge username and password Fixes: https://emqx.atlassian.net/browse/EMQX-9613 --- rel/i18n/emqx_connector_schema_lib.hocon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/i18n/emqx_connector_schema_lib.hocon b/rel/i18n/emqx_connector_schema_lib.hocon index 0e8a2e9a3..2f923c81b 100644 --- a/rel/i18n/emqx_connector_schema_lib.hocon +++ b/rel/i18n/emqx_connector_schema_lib.hocon @@ -13,7 +13,7 @@ database_desc.label: """Database Name""" password.desc: -"""EMQX's password in the external database.""" +"""The password associated with the bridge, used for authentication with the external database.""" password.label: """Password""" @@ -37,7 +37,7 @@ ssl.label: """Enable SSL""" username.desc: -"""EMQX's username in the external database.""" +"""The username associated with the bridge in the external database used for authentication or identification purposes.""" username.label: """Username""" From b1a4f6ea05c0e95e1a2b84ce7affdb9601e982fe Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 15 May 2023 16:37:54 +0200 Subject: [PATCH 095/144] docs: add changelog entry --- changes/ce/fix-10708.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-10708.md diff --git a/changes/ce/fix-10708.md b/changes/ce/fix-10708.md new file mode 100644 index 000000000..617bda24f --- /dev/null +++ b/changes/ce/fix-10708.md @@ -0,0 +1 @@ +Enhanced clarity of the descriptions for the bridge configuration fields (username and password) to better guide users during setup. From 50e7de9db2d55357e17ab69bb928dd92e6dca659 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 16 May 2023 13:30:41 +0800 Subject: [PATCH 096/144] fix(limiter): a bucket with an infinity rate shouldn't be added to limiter server --- .../emqx_limiter/src/emqx_limiter_schema.erl | 1 + .../emqx_limiter/src/emqx_limiter_server.erl | 5 +++-- apps/emqx/test/emqx_ratelimiter_SUITE.erl | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index b9643dae4..64a8bf7a7 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -36,6 +36,7 @@ calc_capacity/1, extract_with_type/2, default_client_config/0, + default_bucket_config/0, short_paths_fields/1, get_listener_opts/1, get_node_opts/1, diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 488f47851..fcb1fd66c 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -131,6 +131,9 @@ connect(Id, Type, Cfg) -> -spec add_bucket(limiter_id(), limiter_type(), hocons:config() | undefined) -> ok. add_bucket(_Id, _Type, undefined) -> ok; +%% a bucket with an infinity rate shouldn't be added to this server, because it is always full +add_bucket(_Id, _Type, #{rate := infinity}) -> + ok; add_bucket(Id, Type, Cfg) -> ?CALL(Type, {add_bucket, Id, Cfg}). @@ -507,8 +510,6 @@ make_root(#{rate := Rate, burst := Burst}) -> correction => 0 }. -do_add_bucket(_Id, #{rate := infinity}, #{root := #{rate := infinity}} = State) -> - State; do_add_bucket(Id, #{rate := Rate} = Cfg, #{buckets := Buckets} = State) -> case maps:get(Id, Buckets, undefined) of undefined -> diff --git a/apps/emqx/test/emqx_ratelimiter_SUITE.erl b/apps/emqx/test/emqx_ratelimiter_SUITE.erl index 6f488eaa9..331fe1b3c 100644 --- a/apps/emqx/test/emqx_ratelimiter_SUITE.erl +++ b/apps/emqx/test/emqx_ratelimiter_SUITE.erl @@ -617,6 +617,24 @@ t_extract_with_type(_) -> ) ). +t_add_bucket(_) -> + Checker = fun(Size) -> + #{buckets := Buckets} = sys:get_state(emqx_limiter_server:whereis(bytes)), + ?assertEqual(Size, maps:size(Buckets), Buckets) + end, + DefBucket = emqx_limiter_schema:default_bucket_config(), + ?assertEqual(ok, emqx_limiter_server:add_bucket(?FUNCTION_NAME, bytes, undefined)), + Checker(0), + ?assertEqual(ok, emqx_limiter_server:add_bucket(?FUNCTION_NAME, bytes, DefBucket)), + Checker(0), + ?assertEqual( + ok, emqx_limiter_server:add_bucket(?FUNCTION_NAME, bytes, DefBucket#{rate := 100}) + ), + Checker(1), + ?assertEqual(ok, emqx_limiter_server:del_bucket(?FUNCTION_NAME, bytes)), + Checker(0), + ok. + %%-------------------------------------------------------------------- %% Test Cases Create Instance %%-------------------------------------------------------------------- From 7d7c069257a3f4a2ee9d74308edc75ad7dde2615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Tue, 16 May 2023 15:32:43 +0800 Subject: [PATCH 097/144] feat: update wehbook's request_timeout into resource_opts --- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge_resource.erl | 26 +++++++--- .../src/schema/emqx_bridge_schema.erl | 49 +------------------ .../src/schema/emqx_bridge_webhook_schema.erl | 28 ++++++++--- 4 files changed, 42 insertions(+), 63 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index e408250be..d2bf0f0c2 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.18"}, + {vsn, "0.1.19"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 0d2feef83..a756f535e 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -165,20 +165,20 @@ create(BridgeId, Conf) -> create(Type, Name, Conf) -> create(Type, Name, Conf, #{}). -create(Type, Name, Conf, Opts0) -> +create(Type, Name, Conf, Opts) -> ?SLOG(info, #{ msg => "create bridge", type => Type, name => Name, config => emqx_utils:redact(Conf) }), - Opts = override_start_after_created(Conf, Opts0), + TypeBin = bin(Type), {ok, _Data} = emqx_resource:create_local( resource_id(Type, Name), <<"emqx_bridge">>, bridge_to_resource_type(Type), - parse_confs(bin(Type), Name, Conf), - Opts + parse_confs(TypeBin, Name, Conf), + parse_opts(TypeBin, Conf, Opts) ), ok. @@ -189,7 +189,7 @@ update(BridgeId, {OldConf, Conf}) -> update(Type, Name, {OldConf, Conf}) -> update(Type, Name, {OldConf, Conf}, #{}). -update(Type, Name, {OldConf, Conf}, Opts0) -> +update(Type, Name, {OldConf, Conf}, Opts) -> %% TODO: sometimes its not necessary to restart the bridge connection. %% %% - if the connection related configs like `servers` is updated, we should restart/start @@ -198,7 +198,6 @@ update(Type, Name, {OldConf, Conf}, Opts0) -> %% the `method` or `headers` of a WebHook is changed, then the bridge can be updated %% without restarting the bridge. %% - Opts = override_start_after_created(Conf, Opts0), case emqx_utils_maps:if_only_to_toggle_enable(OldConf, Conf) of false -> ?SLOG(info, #{ @@ -241,11 +240,12 @@ recreate(Type, Name, Conf) -> recreate(Type, Name, Conf, #{}). recreate(Type, Name, Conf, Opts) -> + TypeBin = bin(Type), emqx_resource:recreate_local( resource_id(Type, Name), bridge_to_resource_type(Type), - parse_confs(bin(Type), Name, Conf), - Opts + parse_confs(TypeBin, Name, Conf), + parse_opts(TypeBin, Conf, Opts) ). create_dry_run(Type, Conf0) -> @@ -402,6 +402,16 @@ bin(Bin) when is_binary(Bin) -> Bin; bin(Str) when is_list(Str) -> list_to_binary(Str); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). +parse_opts(Type, Conf, Opts0) -> + Opts1 = override_start_after_created(Conf, Opts0), + override_resource_request_timeout(Type, Conf, Opts1). + +%% Put webhook's http request_timeout into the resource options +override_resource_request_timeout(<<"webhook">>, #{request_timeout := ReqTimeout}, Opts) -> + Opts#{request_timeout => ReqTimeout}; +override_resource_request_timeout(_Type, _Conf, Opts) -> + Opts. + override_start_after_created(Config, Opts) -> Enabled = maps:get(enable, Config, true), StartAfterCreated = Enabled andalso maps:get(start_after_created, Opts, Enabled), diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index f58805b6b..d1755bf73 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -223,51 +223,6 @@ node_name() -> {"node", mk(binary(), #{desc => ?DESC("desc_node_name"), example => "emqx@127.0.0.1"})}. webhook_bridge_converter(Conf0, _HoconOpts) -> - Conf1 = emqx_bridge_compatible_config:upgrade_pre_ee( + emqx_bridge_compatible_config:upgrade_pre_ee( Conf0, fun emqx_bridge_compatible_config:webhook_maybe_upgrade/1 - ), - case Conf1 of - undefined -> - undefined; - _ -> - maps:map( - fun(_Name, Conf) -> - do_convert_webhook_config(Conf) - end, - Conf1 - ) - end. - -do_convert_webhook_config( - #{<<"request_timeout">> := ReqT, <<"resource_opts">> := #{<<"request_timeout">> := ReqT}} = Conf -) -> - %% ok: same values - Conf; -do_convert_webhook_config( - #{ - <<"request_timeout">> := ReqTRootRaw, - <<"resource_opts">> := #{<<"request_timeout">> := ReqTResourceRaw} - } = Conf0 -) -> - %% different values; we set them to the same, if they are valid - %% durations - MReqTRoot = emqx_schema:to_duration_ms(ReqTRootRaw), - MReqTResource = emqx_schema:to_duration_ms(ReqTResourceRaw), - case {MReqTRoot, MReqTResource} of - {{ok, ReqTRoot}, {ok, ReqTResource}} -> - {_Parsed, ReqTRaw} = max({ReqTRoot, ReqTRootRaw}, {ReqTResource, ReqTResourceRaw}), - Conf1 = emqx_utils_maps:deep_merge( - Conf0, - #{ - <<"request_timeout">> => ReqTRaw, - <<"resource_opts">> => #{<<"request_timeout">> => ReqTRaw} - } - ), - Conf1; - _ -> - %% invalid values; let the type checker complain about - %% that. - Conf0 - end; -do_convert_webhook_config(Conf) -> - Conf. + ). diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl index 1540f77bf..83a3dba9b 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl @@ -40,12 +40,15 @@ fields("put") -> fields("get") -> emqx_bridge_schema:status_fields() ++ fields("post"); fields("creation_opts") -> - lists:filter( - fun({K, _V}) -> - not lists:member(K, unsupported_opts()) - end, - emqx_resource_schema:fields("creation_opts") - ). + [ + deprecated_request_timeout() + | lists:filter( + fun({K, _V}) -> + not lists:member(K, unsupported_opts()) + end, + emqx_resource_schema:fields("creation_opts") + ) + ]. desc("config") -> ?DESC("desc_config"); @@ -163,7 +166,8 @@ unsupported_opts() -> [ enable_batch, batch_size, - batch_time + batch_time, + request_timeout ]. %%====================================================================================== @@ -190,3 +194,13 @@ name_field() -> method() -> enum([post, put, get, delete]). + +deprecated_request_timeout() -> + {request_timeout, + mk( + hoconsc:union([infinity, emqx_schema:duration_ms()]), + #{ + default => <<"15s">>, + deprecated => {since, "5.0.26"} + } + )}. From 1c746ed2894afa9c93ae09e1f55a16abc7ad5225 Mon Sep 17 00:00:00 2001 From: William Yang Date: Sun, 14 May 2023 22:09:49 +0200 Subject: [PATCH 098/144] perf(config): enforcing atom key path in hotcode path --- apps/emqx/src/emqx.erl | 6 +- apps/emqx/src/emqx_config.erl | 57 ++++++++++--------- apps/emqx/src/emqx_shared_sub.erl | 13 ++++- apps/emqx_authn/src/emqx_authn_api.erl | 10 +++- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- .../src/emqx_cluster_rpc_cleaner.erl | 4 +- .../src/emqx_ee_schema_registry.app.src | 2 +- .../src/emqx_ee_schema_registry.erl | 6 +- 8 files changed, 62 insertions(+), 38 deletions(-) diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index ffee5fba7..53e6b9536 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -184,11 +184,13 @@ run_fold_hook(HookPoint, Args, Acc) -> -spec get_config(emqx_utils_maps:config_key_path()) -> term(). get_config(KeyPath) -> - emqx_config:get(KeyPath). + KeyPath1 = emqx_config:ensure_atom_conf_path(KeyPath, {raise_error, config_not_found}), + emqx_config:get(KeyPath1). -spec get_config(emqx_utils_maps:config_key_path(), term()) -> term(). get_config(KeyPath, Default) -> - emqx_config:get(KeyPath, Default). + KeyPath1 = emqx_config:ensure_atom_conf_path(KeyPath, {return, Default}), + emqx_config:get(KeyPath1, Default). -spec get_raw_config(emqx_utils_maps:config_key_path()) -> term(). get_raw_config(KeyPath) -> diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 630952166..91809134c 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -88,6 +88,8 @@ remove_handlers/0 ]). +-export([ensure_atom_conf_path/2]). + -ifdef(TEST). -export([erase_all/0]). -endif. @@ -113,7 +115,8 @@ update_cmd/0, update_args/0, update_error/0, - update_result/0 + update_result/0, + runtime_config_key_path/0 ]). -type update_request() :: term(). @@ -144,6 +147,8 @@ -type config() :: #{atom() => term()} | list() | undefined. -type app_envs() :: [proplists:property()]. +-type runtime_config_key_path() :: [atom()]. + %% @doc For the given path, get root value enclosed in a single-key map. -spec get_root(emqx_utils_maps:config_key_path()) -> map(). get_root([RootName | _]) -> @@ -156,25 +161,21 @@ get_root_raw([RootName | _]) -> %% @doc Get a config value for the given path. %% The path should at least include root config name. --spec get(emqx_utils_maps:config_key_path()) -> term(). +-spec get(runtime_config_key_path()) -> term(). get(KeyPath) -> do_get(?CONF, KeyPath). --spec get(emqx_utils_maps:config_key_path(), term()) -> term(). +-spec get(runtime_config_key_path(), term()) -> term(). get(KeyPath, Default) -> do_get(?CONF, KeyPath, Default). --spec find(emqx_utils_maps:config_key_path()) -> +-spec find(runtime_config_key_path()) -> {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find([]) -> case do_get(?CONF, [], ?CONFIG_NOT_FOUND_MAGIC) of ?CONFIG_NOT_FOUND_MAGIC -> {not_found, []}; Res -> {ok, Res} end; -find(KeyPath) -> - atom_conf_path( - KeyPath, - fun(AtomKeyPath) -> emqx_utils_maps:deep_find(AtomKeyPath, get_root(KeyPath)) end, - {return, {not_found, KeyPath}} - ). +find(AtomKeyPath) -> + emqx_utils_maps:deep_find(AtomKeyPath, get_root(AtomKeyPath)). -spec find_raw(emqx_utils_maps:config_key_path()) -> {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. @@ -712,21 +713,14 @@ do_put(Type, Putter, [RootName | KeyPath], DeepValue) -> NewValue = do_deep_put(Type, Putter, KeyPath, OldValue, DeepValue), persistent_term:put(?PERSIS_KEY(Type, RootName), NewValue). -do_deep_get(?CONF, KeyPath, Map, Default) -> - atom_conf_path( - KeyPath, - fun(AtomKeyPath) -> emqx_utils_maps:deep_get(AtomKeyPath, Map, Default) end, - {return, Default} - ); +do_deep_get(?CONF, AtomKeyPath, Map, Default) -> + emqx_utils_maps:deep_get(AtomKeyPath, Map, Default); do_deep_get(?RAW_CONF, KeyPath, Map, Default) -> emqx_utils_maps:deep_get([bin(Key) || Key <- KeyPath], Map, Default). do_deep_put(?CONF, Putter, KeyPath, Map, Value) -> - atom_conf_path( - KeyPath, - fun(AtomKeyPath) -> Putter(AtomKeyPath, Map, Value) end, - {raise_error, {not_found, KeyPath}} - ); + AtomKeyPath = ensure_atom_conf_path(KeyPath, {raise_error, {not_found, KeyPath}}), + Putter(AtomKeyPath, Map, Value); do_deep_put(?RAW_CONF, Putter, KeyPath, Map, Value) -> Putter([bin(Key) || Key <- KeyPath], Map, Value). @@ -773,15 +767,24 @@ conf_key(?CONF, RootName) -> conf_key(?RAW_CONF, RootName) -> bin(RootName). -atom_conf_path(Path, ExpFun, OnFail) -> - try [atom(Key) || Key <- Path] of - AtomKeyPath -> ExpFun(AtomKeyPath) +ensure_atom_conf_path(Path, OnFail) -> + case lists:all(fun erlang:is_atom/1, Path) of + true -> + %% Do not try to build new atom PATH if it already is. + Path; + _ -> + to_atom_conf_path(Path, OnFail) + end. + +to_atom_conf_path(Path, OnFail) -> + try + [atom(Key) || Key <- Path] catch error:badarg -> case OnFail of - {return, Val} -> - Val; {raise_error, Err} -> - error(Err) + error(Err); + {return, V} -> + V end end. diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 997364898..194e99999 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -158,7 +158,18 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) -> -spec strategy(emqx_topic:group()) -> strategy(). strategy(Group) -> - case emqx:get_config([broker, shared_subscription_group, Group, strategy], undefined) of + case + emqx:get_config( + [ + broker, + shared_subscription_group, + %%binary_to_existing_atom(Group, utf8), + binary_to_atom(Group), + strategy + ], + undefined + ) + of undefined -> emqx:get_config([broker, shared_subscription_strategy]); Strategy -> Strategy end. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index f46718842..8ce3eadf2 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -794,8 +794,8 @@ find_authenticator_config(AuthenticatorID, ConfKeyPath) -> with_listener(ListenerID, Fun) -> case find_listener(ListenerID) of {ok, {BType, BName}} -> - Type = binary_to_existing_atom(BType), - Name = binary_to_existing_atom(BName), + Type = binary_to_existing_atom(BType, utf8), + Name = binary_to_existing_atom(BName, utf8), ChainName = binary_to_atom(ListenerID), Fun(Type, Name, ChainName); {error, Reason} -> @@ -805,7 +805,11 @@ with_listener(ListenerID, Fun) -> find_listener(ListenerID) -> case binary:split(ListenerID, <<":">>) of [BType, BName] -> - case emqx_config:find([listeners, BType, BName]) of + case + emqx_config:find([ + listeners, binary_to_existing_atom(BType), binary_to_existing_atom(BName) + ]) + of {ok, _} -> {ok, {BType, BName}}; {not_found, _, _} -> diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index c7e48990b..e7a1282ce 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -687,7 +687,7 @@ get_metrics_from_local_node(BridgeType, BridgeName) -> ). is_enabled_bridge(BridgeType, BridgeName) -> - try emqx:get_config([bridges, BridgeType, BridgeName]) of + try emqx:get_config([bridges, BridgeType, binary_to_atom(BridgeName)]) of ConfMap -> maps:get(enable, ConfMap, false) catch diff --git a/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl b/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl index fe72cd65b..c558dc908 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl @@ -38,8 +38,8 @@ ]). start_link() -> - MaxHistory = emqx_conf:get(["node", "cluster_call", "max_history"], 100), - CleanupMs = emqx_conf:get(["node", "cluster_call", "cleanup_interval"], 5 * 60 * 1000), + MaxHistory = emqx_conf:get([node, cluster_call, max_history], 100), + CleanupMs = emqx_conf:get([node, cluster_call, cleanup_interval], 5 * 60 * 1000), start_link(MaxHistory, CleanupMs). start_link(MaxHistory, CleanupMs) -> diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src index aa43cf248..b74e9fa7d 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_schema_registry, [ {description, "EMQX Schema Registry"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, [emqx_ee_schema_registry_sup]}, {mod, {emqx_ee_schema_registry_app, []}}, {applications, [ diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl index b1453914b..5ffcb2ba6 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl @@ -58,7 +58,11 @@ get_serde(SchemaName) -> -spec get_schema(schema_name()) -> {ok, map()} | {error, not_found}. get_schema(SchemaName) -> - case emqx_config:get([?CONF_KEY_ROOT, schemas, SchemaName], undefined) of + case + emqx_config:get( + [?CONF_KEY_ROOT, schemas, binary_to_atom(SchemaName)], undefined + ) + of undefined -> {error, not_found}; Config -> From 255f616d261a07c4cb0044003750e825ff5cdcb9 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 16 May 2023 16:36:36 +0800 Subject: [PATCH 099/144] chore: bump emqx app version --- apps/emqx/src/emqx.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 5ca8fc797..be68b438f 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.25"}, + {vsn, "5.0.26"}, {modules, []}, {registered, []}, {applications, [ From 6813ea8e7a76eb5348bda754fa3ff9ae4a2c4d4f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 13:25:33 +0300 Subject: [PATCH 100/144] test(client): dedicate separate testcase to peercert cleaning --- apps/emqx/test/emqx_client_SUITE.erl | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/apps/emqx/test/emqx_client_SUITE.erl b/apps/emqx/test/emqx_client_SUITE.erl index ca5f53070..ed50eb8fe 100644 --- a/apps/emqx/test/emqx_client_SUITE.erl +++ b/apps/emqx/test/emqx_client_SUITE.erl @@ -75,7 +75,8 @@ groups() -> t_username_as_clientid, t_certcn_as_clientid_default_config_tls, t_certcn_as_clientid_tlsv1_3, - t_certcn_as_clientid_tlsv1_2 + t_certcn_as_clientid_tlsv1_2, + t_no_peercert_after_connected ]} ]. @@ -379,6 +380,23 @@ t_certcn_as_clientid_tlsv1_3(_) -> t_certcn_as_clientid_tlsv1_2(_) -> tls_certcn_as_clientid('tlsv1.2'). +t_no_peercert_after_connected(_) -> + emqx_config:put_zone_conf(default, [mqtt], #{}), + ClientId = atom_to_binary(?FUNCTION_NAME), + SslConf = emqx_common_test_helpers:client_ssl_twoway(default), + {ok, Client} = emqtt:start_link([ + {port, 8883}, + {clientid, ClientId}, + {ssl, true}, + {ssl_opts, SslConf} + ]), + {ok, _} = emqtt:connect(Client), + [ConnPid] = emqx_cm:lookup_channels(ClientId), + ?assertMatch( + #{conninfo := ConnInfo} when not is_map_key(peercert, ConnInfo), + emqx_connection:info(ConnPid) + ). + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- @@ -421,10 +439,4 @@ tls_certcn_as_clientid(TLSVsn, RequiredTLSVsn) -> {ok, _} = emqtt:connect(Client), #{clientinfo := #{clientid := CN}} = emqx_cm:get_chan_info(CN), confirm_tls_version(Client, RequiredTLSVsn), - %% verify that the peercert won't be stored in the conninfo - [ChannPid] = emqx_cm:lookup_channels(CN), - SysState = sys:get_state(ChannPid), - ChannelRecord = lists:keyfind(channel, 1, tuple_to_list(SysState)), - ConnInfo = lists:nth(2, tuple_to_list(ChannelRecord)), - ?assertMatch(#{peercert := undefined}, ConnInfo), emqtt:disconnect(Client). From a2aa6b46667630a99a3b2b07a17d85f6f473820f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Tue, 16 May 2023 20:57:57 +0800 Subject: [PATCH 101/144] chore: make ci happy again --- .../test/emqx_bridge_api_SUITE.erl | 14 ++++---- .../emqx_bridge_compatible_config_tests.erl | 34 ++++++++----------- changes/ce/feat-10713.en.md | 3 ++ 3 files changed, 23 insertions(+), 28 deletions(-) create mode 100644 changes/ce/feat-10713.en.md diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index d55b92138..1b5e51a11 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -1284,19 +1284,17 @@ t_inconsistent_webhook_request_timeouts(Config) -> <<"resource_opts">> => #{<<"request_timeout">> => <<"2s">>} } ), - ?assertMatch( - {ok, 201, #{ - %% note: same value on both fields - <<"request_timeout">> := <<"2s">>, - <<"resource_opts">> := #{<<"request_timeout">> := <<"2s">>} - }}, + {ok, 201, #{ + <<"request_timeout">> := <<"1s">>, + <<"resource_opts">> := ResourceOpts + }} = request_json( post, uri(["bridges"]), BadBridgeParams, Config - ) - ), + ), + ?assertNot(maps:is_key(<<"request_timeout">>, ResourceOpts)), ok. %% diff --git a/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl b/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl index acafb84ca..3481ac30c 100644 --- a/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl @@ -59,27 +59,21 @@ webhook_config_test() -> }, check(Conf2) ), - - %% the converter should pick the greater of the two - %% request_timeouts and place them in the root and inside - %% resource_opts. - ?assertMatch( - #{ - <<"bridges">> := #{ - <<"webhook">> := #{ - <<"the_name">> := - #{ - <<"method">> := get, - <<"request_timeout">> := 60_000, - <<"resource_opts">> := #{<<"request_timeout">> := 60_000}, - <<"body">> := <<"${payload}">> - } - } + #{ + <<"bridges">> := #{ + <<"webhook">> := #{ + <<"the_name">> := + #{ + <<"method">> := get, + <<"request_timeout">> := RequestTime, + <<"resource_opts">> := ResourceOpts, + <<"body">> := <<"${payload}">> + } } - }, - check(Conf3) - ), - + } + } = check(Conf3), + ?assertEqual(60_000, RequestTime), + ?assertNot(maps:is_key(<<"requst_timeout">>, ResourceOpts)), ok. up(#{<<"bridges">> := Bridges0} = Conf0) -> diff --git a/changes/ce/feat-10713.en.md b/changes/ce/feat-10713.en.md new file mode 100644 index 000000000..0e28a1a12 --- /dev/null +++ b/changes/ce/feat-10713.en.md @@ -0,0 +1,3 @@ +We deprecated the request_timeout in resource_option of the webhook to keep it consistent with the http request_timeout of the webhook. +From now on, when configuring a webhook through API or configuration files, +it is no longer necessary to configure the request_timeout of the resource. Only configuring the http request_timeout is sufficient, and the request_timeout in the resource will automatically be consistent with the http request_timeout. From 17ab30d8bb8a40450ccd35846e242521da499bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Tue, 16 May 2023 22:04:14 +0800 Subject: [PATCH 102/144] chore: update 10607's log config changelog --- changes/feat-10607.en.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/feat-10607.en.md diff --git a/changes/feat-10607.en.md b/changes/feat-10607.en.md new file mode 100644 index 000000000..a7fef73a3 --- /dev/null +++ b/changes/feat-10607.en.md @@ -0,0 +1,3 @@ +Simplified log configuration: +- Replaced `log.console_handler` with `log.console`, keeping only `enable, level, formatter, and time_offset`. +- Replaced `log.file_handlers` with `log.file`, keeping only `enable, level, formatter, time_offset, rotation_count, rotation_size, to`. From 657df05ad9d762fb6e4bf3c2758d02b31c7ae3cc Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 16 May 2023 09:45:54 -0300 Subject: [PATCH 103/144] fix(buffer_worker): avoid setting flush timer when inflight is full MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes https://emqx.atlassian.net/browse/EMQX-9902 When the buffer worker inflight window is full, we don’t need to set a timer to flush the messages again because there’s no more room, and one of the inflight windows will flush the buffer worker by calling `flush_worker`. Currently, we do set the timer on such situation, and this fact combined with the default batch time of 0 yields a busy loop situation where the CPU spins a lot while inflight messages do not return. --- apps/emqx_resource/src/emqx_resource.app.src | 2 +- apps/emqx_resource/src/emqx_resource_buffer_worker.erl | 3 +-- changes/ce/fix-10717.en.md | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 changes/ce/fix-10717.en.md diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 3e264cb3e..3b92f1200 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.15"}, + {vsn, "0.1.16"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 2dd14c46b..8c5311f2a 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -495,8 +495,7 @@ flush(Data0) -> {keep_state, Data1}; {_, true} -> ?tp(buffer_worker_flush_but_inflight_full, #{}), - Data2 = ensure_flush_timer(Data1), - {keep_state, Data2}; + {keep_state, Data1}; {_, false} -> ?tp(buffer_worker_flush_before_pop, #{}), {Q1, QAckRef, Batch} = replayq:pop(Q0, #{count_limit => BatchSize}), diff --git a/changes/ce/fix-10717.en.md b/changes/ce/fix-10717.en.md new file mode 100644 index 000000000..4c33d6971 --- /dev/null +++ b/changes/ce/fix-10717.en.md @@ -0,0 +1 @@ +Fixed an issue where the buffering layer processes could use a lot of CPU when inflight window is full. From cebde87114d0a36d9b55155bfb2dcd67f1595413 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 15 May 2023 09:05:21 -0300 Subject: [PATCH 104/144] fix(pulsar): use a binary duration as default `health_check_interval` Fixes https://emqx.atlassian.net/browse/EMQX-9885 The frontend needs the default value to match the duration (binary) type to display correctly. --- apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src | 2 +- apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src index b169aa2c4..1665548ae 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_pulsar, [ {description, "EMQX Pulsar Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl index 18faf0e3b..5a87d8a0c 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl @@ -140,7 +140,7 @@ fields(producer_resource_opts) -> lists:filtermap( fun ({health_check_interval = Field, MetaFn}) -> - {true, {Field, override_default(MetaFn, 1_000)}}; + {true, {Field, override_default(MetaFn, <<"1s">>)}}; ({Field, _Meta}) -> lists:member(Field, SupportedOpts) end, From 77cf19c96c0a09ce76dce6215f182bf6c9c0e9f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Tue, 16 May 2023 22:59:08 +0800 Subject: [PATCH 105/144] chore: update 10340's changelog --- changes/ce/fix-10340-en.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changes/ce/fix-10340-en.md diff --git a/changes/ce/fix-10340-en.md b/changes/ce/fix-10340-en.md new file mode 100644 index 000000000..c9ae7b81b --- /dev/null +++ b/changes/ce/fix-10340-en.md @@ -0,0 +1,6 @@ +Fixed the issue that could lead to crash logs being printed when stopping EMQ X via systemd. +``` +2023-03-29T16:43:25.915761+08:00 [error] Generic server memsup terminating. Reason: {port_died,normal}. Last message: {'EXIT',<0.2117.0>,{port_died,normal}}. State: [{data,[{"Timeout",60000}]},{items,{"Memory Usage",[{"Allocated",929959936},{"Total",3832242176}]}},{items,{"Worst Memory User",[{"Pid",<0.2031.0>},{"Memory",4720472}]}}]. +2023-03-29T16:43:25.924764+08:00 [error] crasher: initial call: memsup:init/1, pid: <0.2116.0>, registered_name: memsup, exit: {{port_died,normal},[{gen_server,handle_common_reply,8,[{file,"gen_server.erl"},{line,811}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}]}, ancestors: [os_mon_sup,<0.2114.0>], message_queue_len: 0, messages: [], links: [<0.2115.0>], dictionary: [], trap_exit: true, status: running, heap_size: 4185, stack_size: 29, reductions: 187637; neighbours: +2023-03-29T16:43:25.924979+08:00 [error] Supervisor: {local,os_mon_sup}. Context: child_terminated. Reason: {port_died,normal}. Offender: id=memsup,pid=<0.2116.0>. +``` From 53e961f8b287d12b1e09db155bca266035393a6e Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 18:19:47 +0300 Subject: [PATCH 106/144] ci: ensure dialyzer plt cache is updated on changes --- .github/workflows/run_test_cases.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index bcd84d90c..04a4c0efc 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -109,7 +109,9 @@ jobs: - uses: actions/cache@v3 with: path: "source/emqx_dialyzer_${{ matrix.otp }}_plt" - key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }} + key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('source/rebar.*', 'source/apps/*/rebar.*', 'source/lib-ee/*/rebar.*') }} + restore-keys: | + rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}- - name: run static checks env: PROFILE: ${{ matrix.profile }} From d4b60c561036db0aa0ab6580067d2bc91e290df5 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 16 May 2023 18:59:12 +0300 Subject: [PATCH 107/144] chore(rebalance): improve debug logging --- apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src | 2 +- apps/emqx_node_rebalance/src/emqx_node_rebalance.erl | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src index 381001b87..69cf91f4c 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src @@ -1,6 +1,6 @@ {application, emqx_node_rebalance, [ {description, "EMQX Node Rebalance"}, - {vsn, "5.0.0"}, + {vsn, "5.0.1"}, {registered, [ emqx_node_rebalance_sup, emqx_node_rebalance, diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl index 1f2adc565..70c022308 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl @@ -267,6 +267,9 @@ evict_conns(#{donors := DonorNodes, recipients := RecipientNodes, opts := Opts} ConnEvictRate = maps:get(conn_evict_rate, Opts), NodesToEvict = nodes_to_evict(RecipientAvg, DonorNodeCounts), ?SLOG(warning, #{ + donor_conn_avg => DonorAvg, + recipient_conn_avg => RecipientAvg, + thresholds => Thresholds, msg => "node_rebalance_evict_conns", nodes => NodesToEvict, counts => ConnEvictRate @@ -297,6 +300,9 @@ evict_sessions(#{donors := DonorNodes, recipients := RecipientNodes, opts := Opt SessEvictRate = maps:get(sess_evict_rate, Opts), NodesToEvict = nodes_to_evict(RecipientAvg, DonorNodeCounts), ?SLOG(warning, #{ + donor_sess_avg => DonorAvg, + recipient_sess_avg => RecipientAvg, + thresholds => Thresholds, msg => "node_rebalance_evict_sessions", nodes => NodesToEvict, counts => SessEvictRate From b341a0495541c06d5ebac5a9cab7431bfae39954 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 13:26:22 +0300 Subject: [PATCH 108/144] fix(chan): postpone trimming conninfo until `connected` hooks run Some users expect to get the peer certificate in `connected` hooks, but the `conninfo` was trimmed before `connected` hooks run. --- apps/emqx/src/emqx_channel.erl | 17 +++++++++++++---- apps/emqx/src/emqx_types.erl | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 3fb6a5f6b..8efe02ea6 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -256,9 +256,7 @@ init( ), {NClientInfo, NConnInfo} = take_ws_cookie(ClientInfo, ConnInfo), #channel{ - %% We remove the peercert because it duplicates to what's stored in the socket, - %% Saving a copy here causes unnecessary wast of memory (about 1KB per connection). - conninfo = maps:put(peercert, undefined, NConnInfo), + conninfo = NConnInfo, clientinfo = NClientInfo, topic_aliases = #{ inbound => #{}, @@ -1989,10 +1987,21 @@ ensure_connected( NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, ok = run_hooks('client.connected', [ClientInfo, NConnInfo]), Channel#channel{ - conninfo = NConnInfo, + conninfo = trim_conninfo(NConnInfo), conn_state = connected }. +trim_conninfo(ConnInfo) -> + maps:without( + [ + %% NOTE + %% We remove the peercert because it duplicates what's stored in the socket, + %% otherwise it wastes about 1KB per connection. + peercert + ], + ConnInfo + ). + %%-------------------------------------------------------------------- %% Init Alias Maximum diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index 96d75daba..b1e2c7534 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -129,7 +129,7 @@ socktype := socktype(), sockname := peername(), peername := peername(), - peercert := nossl | undefined | esockd_peercert:peercert(), + peercert => nossl | undefined | esockd_peercert:peercert(), conn_mod := module(), proto_name => binary(), proto_ver => proto_ver(), From 967b2e72e0245d3034d9cb11011b7bf01d311db9 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 13:28:22 +0300 Subject: [PATCH 109/144] test(emqx): remove `peercert` from clientinfo fixtures According to typespec, there's no place for `peercert` in `clientinfo()`. --- apps/emqx/test/emqx_access_control_SUITE.erl | 1 - apps/emqx/test/emqx_channel_SUITE.erl | 1 - apps/emqx/test/emqx_connection_SUITE.erl | 1 - apps/emqx/test/emqx_ws_connection_SUITE.erl | 12 ------------ 4 files changed, 15 deletions(-) diff --git a/apps/emqx/test/emqx_access_control_SUITE.erl b/apps/emqx/test/emqx_access_control_SUITE.erl index 4bf1e05fd..f5fdf223b 100644 --- a/apps/emqx/test/emqx_access_control_SUITE.erl +++ b/apps/emqx/test/emqx_access_control_SUITE.erl @@ -116,7 +116,6 @@ clientinfo(InitProps) -> username => <<"username">>, password => <<"passwd">>, is_superuser => false, - peercert => undefined, mountpoint => undefined }, InitProps diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 2b7280b32..e822bf5e7 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -1211,7 +1211,6 @@ clientinfo(InitProps) -> clientid => <<"clientid">>, username => <<"username">>, is_superuser => false, - peercert => undefined, mountpoint => undefined }, InitProps diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index 0692ec8f5..de3672bf3 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -676,7 +676,6 @@ channel(InitFields) -> clientid => <<"clientid">>, username => <<"username">>, is_superuser => false, - peercert => undefined, mountpoint => undefined }, Conf = emqx_cm:get_session_confs(ClientInfo, #{ diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index 60abe3d3c..bc91bc0ef 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -33,17 +33,6 @@ ] ). --define(STATS_KEYS, [ - recv_oct, - recv_cnt, - send_oct, - send_cnt, - recv_pkt, - recv_msg, - send_pkt, - send_msg -]). - -define(ws_conn, emqx_ws_connection). all() -> emqx_common_test_helpers:all(?MODULE). @@ -618,7 +607,6 @@ channel(InitFields) -> clientid => <<"clientid">>, username => <<"username">>, is_superuser => false, - peercert => undefined, mountpoint => undefined }, Conf = emqx_cm:get_session_confs(ClientInfo, #{ From 74c04b847c75d72ac879dc5ec8b9e15d309f76e6 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 13:41:19 +0300 Subject: [PATCH 110/144] chore: add changelog entry Co-authored-by: Zaiming (Stone) Shi --- changes/ce/fix-10715.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-10715.en.md diff --git a/changes/ce/fix-10715.en.md b/changes/ce/fix-10715.en.md new file mode 100644 index 000000000..68df92df6 --- /dev/null +++ b/changes/ce/fix-10715.en.md @@ -0,0 +1 @@ +Postpone trimming the connection information structure until after `client.connected` hooks have been executed. These hooks once again have access to the client's peer certificate. From 3cd95f40e5a6df69397cbb761b2839abe5d83668 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 18:05:43 +0300 Subject: [PATCH 111/144] test(chan): verify hooks receive peercert until connected --- apps/emqx/include/asserts.hrl | 30 ++++++++++++++++++++++++++++ apps/emqx/test/emqx_client_SUITE.erl | 27 ++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/apps/emqx/include/asserts.hrl b/apps/emqx/include/asserts.hrl index 98d8e72fc..4936da1f9 100644 --- a/apps/emqx/include/asserts.hrl +++ b/apps/emqx/include/asserts.hrl @@ -29,3 +29,33 @@ ) ) ). + +-define(drainMailbox(), + (fun F__Flush_() -> + receive + X__Msg_ -> [X__Msg_ | F__Flush_()] + after 0 -> [] + end + end)() +). + +-define(assertReceive(PATTERN), + ?assertReceive(PATTERN, 1000) +). + +-define(assertReceive(PATTERN, TIMEOUT), + (fun() -> + receive + X__V = PATTERN -> X__V + after TIMEOUT -> + erlang:error( + {assertReceive, [ + {module, ?MODULE}, + {line, ?LINE}, + {expression, (??PATTERN)}, + {mailbox, ?drainMailbox()} + ]} + ) + end + end)() +). diff --git a/apps/emqx/test/emqx_client_SUITE.erl b/apps/emqx/test/emqx_client_SUITE.erl index ed50eb8fe..14617a152 100644 --- a/apps/emqx/test/emqx_client_SUITE.erl +++ b/apps/emqx/test/emqx_client_SUITE.erl @@ -22,6 +22,8 @@ -import(lists, [nth/2]). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("emqx/include/asserts.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -76,7 +78,7 @@ groups() -> t_certcn_as_clientid_default_config_tls, t_certcn_as_clientid_tlsv1_3, t_certcn_as_clientid_tlsv1_2, - t_no_peercert_after_connected + t_peercert_preserved_before_connected ]} ]. @@ -380,8 +382,18 @@ t_certcn_as_clientid_tlsv1_3(_) -> t_certcn_as_clientid_tlsv1_2(_) -> tls_certcn_as_clientid('tlsv1.2'). -t_no_peercert_after_connected(_) -> - emqx_config:put_zone_conf(default, [mqtt], #{}), +t_peercert_preserved_before_connected(_) -> + ok = emqx_config:put_zone_conf(default, [mqtt], #{}), + ok = emqx_hooks:add( + 'client.connect', + {?MODULE, on_hook, ['client.connect', self()]}, + ?HP_HIGHEST + ), + ok = emqx_hooks:add( + 'client.connected', + {?MODULE, on_hook, ['client.connected', self()]}, + ?HP_HIGHEST + ), ClientId = atom_to_binary(?FUNCTION_NAME), SslConf = emqx_common_test_helpers:client_ssl_twoway(default), {ok, Client} = emqtt:start_link([ @@ -391,12 +403,21 @@ t_no_peercert_after_connected(_) -> {ssl_opts, SslConf} ]), {ok, _} = emqtt:connect(Client), + _ = ?assertReceive({'client.connect', #{peercert := PC}} when is_binary(PC)), + _ = ?assertReceive({'client.connected', #{peercert := PC}} when is_binary(PC)), [ConnPid] = emqx_cm:lookup_channels(ClientId), ?assertMatch( #{conninfo := ConnInfo} when not is_map_key(peercert, ConnInfo), emqx_connection:info(ConnPid) ). +on_hook(ConnInfo, _, 'client.connect' = HP, Pid) -> + _ = Pid ! {HP, ConnInfo}, + ok; +on_hook(_ClientInfo, ConnInfo, 'client.connected' = HP, Pid) -> + _ = Pid ! {HP, ConnInfo}, + ok. + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- From 85089a32109bd6057181fc07e52b6fb3e1d03bdc Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 16 May 2023 17:15:42 -0300 Subject: [PATCH 112/144] fix(buffer_worker): correctly flush the buffer workers when inflight table room is made The previous commit uncovered another bug that was hidden by it: `maybe_flush_after_async_reply` was sending a message to the wrong PID. It was sending a message to `self()` meaning to target a buffer worker, but `self()` in that context is never the buffer worker, it's the connector's worker. This change also revealed a race condition where the buffer workers could stop flushing messages. So we piggy-backed on the atomic update of the table size count to check if the buffer worker should be poked to continue flushing. This allows us to get rid of `maybe_flush_after_async_reply` altogether. --- .../src/emqx_resource_buffer_worker.erl | 102 +++++++++--------- .../test/emqx_resource_SUITE.erl | 6 +- 2 files changed, 57 insertions(+), 51 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 8c5311f2a..6145c3d87 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -70,18 +70,6 @@ -define(RETRY_IDX, 3). -define(WORKER_MREF_IDX, 4). --define(ENSURE_ASYNC_FLUSH(InflightTID, EXPR), - (fun() -> - IsFullBefore = is_inflight_full(InflightTID), - case (EXPR) of - blocked -> - ok; - ok -> - ok = maybe_flush_after_async_reply(IsFullBefore) - end - end)() -). - -type id() :: binary(). -type index() :: pos_integer(). -type expire_at() :: infinity | integer(). @@ -337,7 +325,8 @@ resume_from_blocked(Data) -> {next_state, running, Data} end; {expired, Ref, Batch} -> - IsAcked = ack_inflight(InflightTID, Ref, Id, Index), + WorkerPid = self(), + IsAcked = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), IsAcked andalso emqx_resource_metrics:dropped_expired_inc(Id, length(Batch)), ?tp(buffer_worker_retry_expired, #{expired => Batch}), resume_from_blocked(Data); @@ -389,7 +378,8 @@ retry_inflight_sync(Ref, QueryOrBatch, Data0) -> {keep_state, Data0, {state_timeout, ResumeT, unblock}}; %% Send ok or failed but the resource is working {ack, PostFn} -> - IsAcked = ack_inflight(InflightTID, Ref, Id, Index), + WorkerPid = self(), + IsAcked = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), %% we need to defer bumping the counters after %% `inflight_drop' to avoid the race condition when an %% inflight request might get completed concurrently with @@ -595,13 +585,14 @@ do_flush( %% must ensure the async worker is being monitored for %% such requests. IsUnrecoverableError = is_unrecoverable_error(Result), + WorkerPid = self(), case is_async_return(Result) of true when IsUnrecoverableError -> - ack_inflight(InflightTID, Ref, Id, Index); + ack_inflight(InflightTID, Ref, Id, Index, WorkerPid); true -> ok; false -> - ack_inflight(InflightTID, Ref, Id, Index) + ack_inflight(InflightTID, Ref, Id, Index, WorkerPid) end, {Data1, WorkerMRef} = ensure_async_worker_monitored(Data0, Result), store_async_worker_reference(InflightTID, Ref, WorkerMRef), @@ -679,13 +670,14 @@ do_flush(#{queue := Q1} = Data0, #{ %% must ensure the async worker is being monitored for %% such requests. IsUnrecoverableError = is_unrecoverable_error(Result), + WorkerPid = self(), case is_async_return(Result) of true when IsUnrecoverableError -> - ack_inflight(InflightTID, Ref, Id, Index); + ack_inflight(InflightTID, Ref, Id, Index, WorkerPid); true -> ok; false -> - ack_inflight(InflightTID, Ref, Id, Index) + ack_inflight(InflightTID, Ref, Id, Index, WorkerPid) end, {Data1, WorkerMRef} = ensure_async_worker_monitored(Data0, Result), store_async_worker_reference(InflightTID, Ref, WorkerMRef), @@ -1005,7 +997,7 @@ handle_async_reply( discard -> ok; continue -> - ?ENSURE_ASYNC_FLUSH(InflightTID, handle_async_reply1(ReplyContext, Result)) + handle_async_reply1(ReplyContext, Result) end. handle_async_reply1( @@ -1014,6 +1006,7 @@ handle_async_reply1( inflight_tid := InflightTID, resource_id := Id, worker_index := Index, + buffer_worker := WorkerPid, min_query := ?QUERY(_, _, _, ExpireAt) = _Query } = ReplyContext, Result @@ -1025,7 +1018,7 @@ handle_async_reply1( Now = now_(), case is_expired(ExpireAt, Now) of true -> - IsAcked = ack_inflight(InflightTID, Ref, Id, Index), + IsAcked = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), IsAcked andalso emqx_resource_metrics:late_reply_inc(Id), ?tp(handle_async_reply_expired, #{expired => [_Query]}), ok; @@ -1039,7 +1032,7 @@ do_handle_async_reply( resource_id := Id, request_ref := Ref, worker_index := Index, - buffer_worker := Pid, + buffer_worker := WorkerPid, inflight_tid := InflightTID, min_query := ?QUERY(ReplyTo, _, Sent, _ExpireAt) = _Query }, @@ -1062,10 +1055,10 @@ do_handle_async_reply( nack -> %% Keep retrying. ok = mark_inflight_as_retriable(InflightTID, Ref), - ok = ?MODULE:block(Pid), + ok = ?MODULE:block(WorkerPid), blocked; ack -> - ok = do_async_ack(InflightTID, Ref, Id, Index, PostFn, QueryOpts) + ok = do_async_ack(InflightTID, Ref, Id, Index, WorkerPid, PostFn, QueryOpts) end. handle_async_batch_reply( @@ -1080,7 +1073,7 @@ handle_async_batch_reply( discard -> ok; continue -> - ?ENSURE_ASYNC_FLUSH(InflightTID, handle_async_batch_reply1(ReplyContext, Result)) + handle_async_batch_reply1(ReplyContext, Result) end. handle_async_batch_reply1( @@ -1118,6 +1111,7 @@ handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> #{ resource_id := Id, worker_index := Index, + buffer_worker := WorkerPid, inflight_tid := InflightTID, request_ref := Ref, min_batch := Batch @@ -1140,7 +1134,7 @@ handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> case RealNotExpired of [] -> %% all expired, no need to update back the inflight batch - _ = ack_inflight(InflightTID, Ref, Id, Index), + _ = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), ok; _ -> %% some queries are not expired, put them back to the inflight batch @@ -1151,7 +1145,7 @@ handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> do_handle_async_batch_reply( #{ - buffer_worker := Pid, + buffer_worker := WorkerPid, resource_id := Id, worker_index := Index, inflight_tid := InflightTID, @@ -1172,14 +1166,14 @@ do_handle_async_batch_reply( nack -> %% Keep retrying. ok = mark_inflight_as_retriable(InflightTID, Ref), - ok = ?MODULE:block(Pid), + ok = ?MODULE:block(WorkerPid), blocked; ack -> - ok = do_async_ack(InflightTID, Ref, Id, Index, PostFn, QueryOpts) + ok = do_async_ack(InflightTID, Ref, Id, Index, WorkerPid, PostFn, QueryOpts) end. -do_async_ack(InflightTID, Ref, Id, Index, PostFn, QueryOpts) -> - IsKnownRef = ack_inflight(InflightTID, Ref, Id, Index), +do_async_ack(InflightTID, Ref, Id, Index, WorkerPid, PostFn, QueryOpts) -> + IsKnownRef = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), case maps:get(simple_query, QueryOpts, false) of true -> PostFn(); @@ -1190,18 +1184,6 @@ do_async_ack(InflightTID, Ref, Id, Index, PostFn, QueryOpts) -> end, ok. -maybe_flush_after_async_reply(_WasFullBeforeReplyHandled = false) -> - %% inflight was not full before async reply is handled, - %% after it is handled, the inflight table must be even smaller - %% hance we can rely on the buffer worker's flush timer to trigger - %% the next flush - ?tp(skip_flushing_worker, #{}), - ok; -maybe_flush_after_async_reply(_WasFullBeforeReplyHandled = true) -> - %% the inflight table was full before handling aync reply - ?tp(do_flushing_worker, #{}), - ok = ?MODULE:flush_worker(self()). - %% check if the async reply is valid. %% e.g. if a connector evaluates the callback more than once: %% 1. If the request was previously deleted from inflight table due to @@ -1428,9 +1410,9 @@ store_async_worker_reference(InflightTID, Ref, WorkerMRef) when ), ok. -ack_inflight(undefined, _Ref, _Id, _Index) -> +ack_inflight(undefined, _Ref, _Id, _Index, _WorkerPid) -> false; -ack_inflight(InflightTID, Ref, Id, Index) -> +ack_inflight(InflightTID, Ref, Id, Index, WorkerPid) -> {Count, Removed} = case ets:take(InflightTID, Ref) of [?INFLIGHT_ITEM(Ref, ?QUERY(_, _, _, _), _IsRetriable, _WorkerMRef)] -> @@ -1440,7 +1422,11 @@ ack_inflight(InflightTID, Ref, Id, Index) -> [] -> {0, false} end, - ok = dec_inflight_remove(InflightTID, Count, Removed), + FlushCheck = dec_inflight_remove(InflightTID, Count, Removed), + case FlushCheck of + continue -> ok; + flush -> ?MODULE:flush_worker(WorkerPid) + end, IsKnownRef = (Count > 0), case IsKnownRef of true -> @@ -1475,16 +1461,32 @@ inc_inflight(InflightTID, Count) -> _ = ets:update_counter(InflightTID, ?BATCH_COUNT_REF, {2, 1}), ok. +-spec dec_inflight_remove(undefined | ets:tid(), non_neg_integer(), Removed :: boolean()) -> + continue | flush. dec_inflight_remove(_InflightTID, _Count = 0, _Removed = false) -> - ok; + continue; dec_inflight_remove(InflightTID, _Count = 0, _Removed = true) -> - _ = ets:update_counter(InflightTID, ?BATCH_COUNT_REF, {2, -1, 0, 0}), - ok; + NewValue = ets:update_counter(InflightTID, ?BATCH_COUNT_REF, {2, -1, 0, 0}), + MaxValue = emqx_utils_ets:lookup_value(InflightTID, ?MAX_SIZE_REF, 0), + %% if the new value is Max - 1, it means that we've just made room + %% in the inflight table, so we should poke the buffer worker to + %% make it continue flushing. + case NewValue =:= MaxValue - 1 of + true -> flush; + false -> continue + end; dec_inflight_remove(InflightTID, Count, _Removed = true) when Count > 0 -> %% If Count > 0, it must have been removed - _ = ets:update_counter(InflightTID, ?BATCH_COUNT_REF, {2, -1, 0, 0}), + NewValue = ets:update_counter(InflightTID, ?BATCH_COUNT_REF, {2, -1, 0, 0}), _ = ets:update_counter(InflightTID, ?SIZE_REF, {2, -Count, 0, 0}), - ok. + MaxValue = emqx_utils_ets:lookup_value(InflightTID, ?MAX_SIZE_REF, 0), + %% if the new value is Max - 1, it means that we've just made room + %% in the inflight table, so we should poke the buffer worker to + %% make it continue flushing. + case NewValue =:= MaxValue - 1 of + true -> flush; + false -> continue + end. dec_inflight_update(_InflightTID, _Count = 0) -> ok; diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 809f101a8..fc338b512 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1627,7 +1627,11 @@ t_retry_async_inflight_full(_Config) -> end ] ), - ?assertEqual(0, emqx_resource_metrics:inflight_get(?ID)), + ?retry( + _Sleep = 300, + _Attempts0 = 20, + ?assertEqual(0, emqx_resource_metrics:inflight_get(?ID)) + ), ok. %% this test case is to ensure the buffer worker will not go crazy even From 18043150bea8c294772f96949cf4fd3ae5b42d70 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 17 May 2023 10:55:47 +0800 Subject: [PATCH 113/144] fix: cannot access columns exported by FOREACH in DO clause --- apps/emqx_rule_engine/src/emqx_rule_runtime.erl | 4 ++-- .../emqx_rule_engine/test/emqx_rule_engine_SUITE.erl | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index d7412d03c..5b7f962fb 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -144,14 +144,14 @@ do_apply_rule( ) of true -> - Collection2 = filter_collection(Columns, InCase, DoEach, Collection), + Collection2 = filter_collection(ColumnsAndSelected, InCase, DoEach, Collection), case Collection2 of [] -> ok = emqx_metrics_worker:inc(rule_metrics, RuleId, 'failed.no_result'); _ -> ok = emqx_metrics_worker:inc(rule_metrics, RuleId, 'passed') end, - NewEnvs = maps:merge(Columns, Envs), + NewEnvs = maps:merge(ColumnsAndSelected, Envs), {ok, [handle_action_list(RuleId, Actions, Coll, NewEnvs) || Coll <- Collection2]}; false -> ok = emqx_metrics_worker:inc(rule_metrics, RuleId, 'failed.no_result'), diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index eb253e516..9c3e5513a 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -1735,11 +1735,12 @@ t_sqlparse_foreach_7(_Config) -> ) ). +-define(COLL, #{<<"info">> := [<<"haha">>, #{<<"name">> := <<"cmd1">>, <<"cmd">> := <<"1">>}]}). t_sqlparse_foreach_8(_Config) -> %% Verify foreach-do-incase and cascaded AS Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " - "do info.cmd as msg_type, info.name as name " + "do info.cmd as msg_type, info.name as name, s, c " "incase is_map(info) " "from \"t/#\" " "where s.page = '2' ", @@ -1748,7 +1749,14 @@ t_sqlparse_foreach_8(_Config) -> "{\"info\":[\"haha\", {\"name\":\"cmd1\", \"cmd\":\"1\"}]} } }" >>, ?assertMatch( - {ok, [#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, + {ok, [ + #{ + <<"name">> := <<"cmd1">>, + <<"msg_type">> := <<"1">>, + <<"s">> := #{<<"page">> := 2, <<"collection">> := ?COLL}, + <<"c">> := ?COLL + } + ]}, emqx_rule_sqltester:test( #{ sql => Sql, From 1f7ede90a429c7a568965d688de3cfe0fef75cb0 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 17 May 2023 11:01:31 +0800 Subject: [PATCH 114/144] chore: update app version && changes --- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- changes/ce/fix-10728.en.md | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 changes/ce/fix-10728.en.md diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 94a48fb35..8dc78958a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.16"}, + {vsn, "5.0.17"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]}, diff --git a/changes/ce/fix-10728.en.md b/changes/ce/fix-10728.en.md new file mode 100644 index 000000000..82bd123e9 --- /dev/null +++ b/changes/ce/fix-10728.en.md @@ -0,0 +1,11 @@ +Fixed an issue where the rule engine was unable to access variables exported by `FOREACH` in the `DO` clause. + + Given a payload: `{"date": "2023-05-06", "array": ["a"]}`, as well as the following SQL statement: + ``` + FOREACH payload.date as date, payload.array as elem + DO date, elem + FROM "t/#" + ``` + Prior to the fix, the `date` variable exported by `FOREACH` could not be accessed in the `DO` clause of the above SQL, resulting in the following output for the SQL statement: + `[{"elem": "a","date": "undefined"}]`. + After the fix, the output of the SQL statement is: `[{"elem": "a","date": "2023-05-06"}]` From 2b99a9b988c336345a1a19bfe98de583a8c01666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Wed, 17 May 2023 13:31:45 +0800 Subject: [PATCH 115/144] feat: hide resource_opts's request_timeout --- apps/emqx_bridge/src/emqx_bridge_api.erl | 13 +++++++--- apps/emqx_bridge/src/emqx_bridge_resource.erl | 15 ++++-------- .../src/schema/emqx_bridge_schema.erl | 23 ++++++++++++++++-- .../src/schema/emqx_bridge_webhook_schema.erl | 8 +++---- .../test/emqx_bridge_api_SUITE.erl | 24 +++++++++++++++++++ .../emqx_bridge_compatible_config_tests.erl | 2 +- changes/ce/feat-10713.en.md | 2 +- 7 files changed, 65 insertions(+), 22 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index c7e48990b..9802d5fe8 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -892,11 +892,18 @@ fill_defaults(Type, RawConf) -> pack_bridge_conf(Type, RawConf) -> #{<<"bridges">> => #{bin(Type) => #{<<"foo">> => RawConf}}}. -unpack_bridge_conf(Type, PackedConf) -> - #{<<"bridges">> := Bridges} = PackedConf, - #{<<"foo">> := RawConf} = maps:get(bin(Type), Bridges), +%% Hide webhook's resource_opts.request_timeout from user. +filter_raw_conf(<<"webhook">>, RawConf0) -> + emqx_utils_maps:deep_remove([<<"resource_opts">>, <<"request_timeout">>], RawConf0); +filter_raw_conf(_TypeBin, RawConf) -> RawConf. +unpack_bridge_conf(Type, PackedConf) -> + TypeBin = bin(Type), + #{<<"bridges">> := Bridges} = PackedConf, + #{<<"foo">> := RawConf} = maps:get(TypeBin, Bridges), + filter_raw_conf(TypeBin, RawConf). + is_ok(ok) -> ok; is_ok(OkResult = {ok, _}) -> diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index a756f535e..60ee242d1 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -178,7 +178,7 @@ create(Type, Name, Conf, Opts) -> <<"emqx_bridge">>, bridge_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), - parse_opts(TypeBin, Conf, Opts) + parse_opts(Conf, Opts) ), ok. @@ -245,7 +245,7 @@ recreate(Type, Name, Conf, Opts) -> resource_id(Type, Name), bridge_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), - parse_opts(TypeBin, Conf, Opts) + parse_opts(Conf, Opts) ). create_dry_run(Type, Conf0) -> @@ -402,15 +402,8 @@ bin(Bin) when is_binary(Bin) -> Bin; bin(Str) when is_list(Str) -> list_to_binary(Str); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). -parse_opts(Type, Conf, Opts0) -> - Opts1 = override_start_after_created(Conf, Opts0), - override_resource_request_timeout(Type, Conf, Opts1). - -%% Put webhook's http request_timeout into the resource options -override_resource_request_timeout(<<"webhook">>, #{request_timeout := ReqTimeout}, Opts) -> - Opts#{request_timeout => ReqTimeout}; -override_resource_request_timeout(_Type, _Conf, Opts) -> - Opts. +parse_opts(Conf, Opts0) -> + override_start_after_created(Conf, Opts0). override_start_after_created(Config, Opts) -> Enabled = maps:get(enable, Config, true), diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index d1755bf73..b590f0cd4 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -223,6 +223,25 @@ node_name() -> {"node", mk(binary(), #{desc => ?DESC("desc_node_name"), example => "emqx@127.0.0.1"})}. webhook_bridge_converter(Conf0, _HoconOpts) -> - emqx_bridge_compatible_config:upgrade_pre_ee( + Conf1 = emqx_bridge_compatible_config:upgrade_pre_ee( Conf0, fun emqx_bridge_compatible_config:webhook_maybe_upgrade/1 - ). + ), + case Conf1 of + undefined -> + undefined; + _ -> + maps:map( + fun(_Name, Conf) -> + do_convert_webhook_config(Conf) + end, + Conf1 + ) + end. + +%% We hide resource_opts.request_timeout from user. +do_convert_webhook_config( + #{<<"request_timeout">> := ReqT, <<"resource_opts">> := ResOpts} = Conf +) -> + Conf#{<<"resource_opts">> => ResOpts#{<<"request_timeout">> => ReqT}}; +do_convert_webhook_config(Conf) -> + Conf. diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl index 83a3dba9b..de1e09abb 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl @@ -41,7 +41,7 @@ fields("get") -> emqx_bridge_schema:status_fields() ++ fields("post"); fields("creation_opts") -> [ - deprecated_request_timeout() + hidden_request_timeout() | lists:filter( fun({K, _V}) -> not lists:member(K, unsupported_opts()) @@ -195,12 +195,12 @@ name_field() -> method() -> enum([post, put, get, delete]). -deprecated_request_timeout() -> +hidden_request_timeout() -> {request_timeout, mk( hoconsc:union([infinity, emqx_schema:duration_ms()]), #{ - default => <<"15s">>, - deprecated => {since, "5.0.26"} + required => false, + importance => ?IMPORTANCE_HIDDEN } )}. diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 1b5e51a11..288b1da29 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -1295,8 +1295,32 @@ t_inconsistent_webhook_request_timeouts(Config) -> Config ), ?assertNot(maps:is_key(<<"request_timeout">>, ResourceOpts)), + validate_resource_request_timeout(proplists:get_value(group, Config), 1000, Name), ok. +validate_resource_request_timeout(single, Timeout, Name) -> + SentData = #{payload => <<"Hello EMQX">>, timestamp => 1668602148000}, + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), + ResId = emqx_bridge_resource:resource_id(<<"webhook">>, Name), + ?check_trace( + begin + {ok, Res} = + ?wait_async_action( + emqx_bridge:send_message(BridgeID, SentData), + #{?snk_kind := async_query}, + 1000 + ), + ?assertMatch({ok, #{id := ResId, query_opts := #{timeout := Timeout}}}, Res) + end, + fun(Trace0) -> + Trace = ?of_kind(async_query, Trace0), + ?assertMatch([#{query_opts := #{timeout := Timeout}}], Trace), + ok + end + ); +validate_resource_request_timeout(_Cluster, _Timeout, _Name) -> + ignore. + %% request(Method, URL, Config) -> diff --git a/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl b/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl index 3481ac30c..080ca9107 100644 --- a/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl @@ -73,7 +73,7 @@ webhook_config_test() -> } } = check(Conf3), ?assertEqual(60_000, RequestTime), - ?assertNot(maps:is_key(<<"requst_timeout">>, ResourceOpts)), + ?assertMatch(#{<<"request_timeout">> := 60_000}, ResourceOpts), ok. up(#{<<"bridges">> := Bridges0} = Conf0) -> diff --git a/changes/ce/feat-10713.en.md b/changes/ce/feat-10713.en.md index 0e28a1a12..6de542be6 100644 --- a/changes/ce/feat-10713.en.md +++ b/changes/ce/feat-10713.en.md @@ -1,3 +1,3 @@ -We deprecated the request_timeout in resource_option of the webhook to keep it consistent with the http request_timeout of the webhook. +We hide the request_timeout in resource_option of the webhook to keep it consistent with the http request_timeout of the webhook. From now on, when configuring a webhook through API or configuration files, it is no longer necessary to configure the request_timeout of the resource. Only configuring the http request_timeout is sufficient, and the request_timeout in the resource will automatically be consistent with the http request_timeout. From bce43978a2e1d88a0e6033156b3c501371352134 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 9 May 2023 15:26:52 +0200 Subject: [PATCH 116/144] refactor: the clickhouse bridge to use the new bridge app structure Fixes: https://emqx.atlassian.net/browse/EMQX-9538 --- apps/emqx_bridge_clickhouse/README.md | 2 +- apps/emqx_bridge_clickhouse/docker-ct | 1 + apps/emqx_bridge_clickhouse/rebar.config | 11 +++++++++++ .../src/emqx_bridge_clickhouse.app.src | 4 ++-- .../src/emqx_bridge_clickhouse.erl | 4 ++-- .../src/emqx_bridge_clickhouse_connector.erl | 2 +- .../test/emqx_bridge_clickhouse_SUITE.erl | 5 ++--- .../emqx_bridge_clickhouse_connector_SUITE.erl | 15 +++++++++++---- changes/ee/refactor-10654.en.md | 1 + lib-ee/emqx_ee_bridge/docker-ct | 1 - lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src | 3 ++- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 8 ++++---- lib-ee/emqx_ee_connector/docker-ct | 1 - lib-ee/emqx_ee_connector/rebar.config | 1 - .../src/emqx_ee_connector.app.src | 3 +-- mix.exs | 4 ++-- ...ckhouse.hocon => emqx_bridge_clickhouse.hocon} | 2 +- ...con => emqx_bridge_clickhouse_connector.hocon} | 2 +- ...ckhouse.hocon => emqx_bridge_clickhouse.hocon} | 2 +- ...con => emqx_bridge_clickhouse_connector.hocon} | 2 +- 20 files changed, 45 insertions(+), 29 deletions(-) create mode 100644 apps/emqx_bridge_clickhouse/docker-ct create mode 100644 apps/emqx_bridge_clickhouse/rebar.config rename lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl => apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl (97%) rename lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl => apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl (99%) rename lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_clickhouse_SUITE.erl => apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl (98%) rename lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl => apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl (93%) create mode 100644 changes/ee/refactor-10654.en.md rename rel/i18n/{emqx_ee_bridge_clickhouse.hocon => emqx_bridge_clickhouse.hocon} (98%) rename rel/i18n/{emqx_ee_connector_clickhouse.hocon => emqx_bridge_clickhouse_connector.hocon} (89%) rename rel/i18n/zh/{emqx_ee_bridge_clickhouse.hocon => emqx_bridge_clickhouse.hocon} (97%) rename rel/i18n/zh/{emqx_ee_connector_clickhouse.hocon => emqx_bridge_clickhouse_connector.hocon} (88%) diff --git a/apps/emqx_bridge_clickhouse/README.md b/apps/emqx_bridge_clickhouse/README.md index ff870e87d..e74afab3d 100644 --- a/apps/emqx_bridge_clickhouse/README.md +++ b/apps/emqx_bridge_clickhouse/README.md @@ -23,7 +23,7 @@ User can create a rule and easily ingest IoT data into ClickHouse by leveraging - Several APIs are provided for bridge management, which includes create bridge, update bridge, get bridge, stop or restart bridge and list bridges etc. - Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) +- Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) for more detailed information. diff --git a/apps/emqx_bridge_clickhouse/docker-ct b/apps/emqx_bridge_clickhouse/docker-ct new file mode 100644 index 000000000..938ba95c6 --- /dev/null +++ b/apps/emqx_bridge_clickhouse/docker-ct @@ -0,0 +1 @@ +clickhouse diff --git a/apps/emqx_bridge_clickhouse/rebar.config b/apps/emqx_bridge_clickhouse/rebar.config new file mode 100644 index 000000000..a8da74b43 --- /dev/null +++ b/apps/emqx_bridge_clickhouse/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang; -*- +{erl_opts, [debug_info]}. +{deps, [ {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}} + , {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + {apps, [emqx_bridge_clickhouse]} +]}. diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src index a0b409d5b..751628fa4 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src @@ -1,8 +1,8 @@ {application, emqx_bridge_clickhouse, [ {description, "EMQX Enterprise ClickHouse Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.2.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, clickhouse]}, {env, []}, {modules, []}, {links, []} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl similarity index 97% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl rename to apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl index 56671c586..d317ecc5c 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_clickhouse). +-module(emqx_bridge_clickhouse). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). -include_lib("typerefl/include/types.hrl"). @@ -101,7 +101,7 @@ fields("config") -> } )} ] ++ - emqx_ee_connector_clickhouse:fields(config); + emqx_bridge_clickhouse_connector:fields(config); fields("creation_opts") -> emqx_resource_schema:fields("creation_opts"); fields("post") -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl similarity index 99% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl rename to apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl index a7afcd6d5..a2de1d3c9 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_clickhouse). +-module(emqx_bridge_clickhouse_connector). -include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_clickhouse_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl similarity index 98% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_clickhouse_SUITE.erl rename to apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl index 6d4762882..b39296044 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_clickhouse_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl @@ -2,17 +2,16 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_clickhouse_SUITE). +-module(emqx_bridge_clickhouse_SUITE). -compile(nowarn_export_all). -compile(export_all). -define(CLICKHOUSE_HOST, "clickhouse"). --define(CLICKHOUSE_RESOURCE_MOD, emqx_ee_connector_clickhouse). -include_lib("emqx_connector/include/emqx_connector.hrl"). %% See comment in -%% lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl for how to +%% lib-ee/emqx_ee_connector/test/ee_bridge_clickhouse_connector_SUITE.erl for how to %% run this without bringing up the whole CI infrastucture %%------------------------------------------------------------------------------ diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl similarity index 93% rename from lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl rename to apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl index e704a2c0c..549a0d870 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_clickhouse_SUITE). +-module(emqx_bridge_clickhouse_connector_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -13,7 +13,7 @@ -include_lib("stdlib/include/assert.hrl"). -define(CLICKHOUSE_HOST, "clickhouse"). --define(CLICKHOUSE_RESOURCE_MOD, emqx_ee_connector_clickhouse). +-define(CLICKHOUSE_RESOURCE_MOD, emqx_bridge_clickhouse_connector). %% This test SUITE requires a running clickhouse instance. If you don't want to %% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script @@ -21,7 +21,15 @@ %% from root of the EMQX directory.). You also need to set ?CLICKHOUSE_HOST and %% ?CLICKHOUSE_PORT to appropriate values. %% -%% docker run -d -p 18123:8123 -p19000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 -v "`pwd`/.ci/docker-compose-file/clickhouse/users.xml:/etc/clickhouse-server/users.xml" -v "`pwd`/.ci/docker-compose-file/clickhouse/config.xml:/etc/clickhouse-server/config.xml" clickhouse/clickhouse-server +%% docker run \ +%% -d \ +%% -p 18123:8123 \ +%% -p 19000:9000 \ +%% --name some-clickhouse-server \ +%% --ulimit nofile=262144:262144 \ +%% -v "`pwd`/.ci/docker-compose-file/clickhouse/users.xml:/etc/clickhouse-server/users.xml" \ +%% -v "`pwd`/.ci/docker-compose-file/clickhouse/config.xml:/etc/clickhouse-server/config.xml" \ +%% clickhouse/clickhouse-server all() -> emqx_common_test_helpers:all(?MODULE). @@ -119,7 +127,6 @@ perform_lifecycle_check(ResourceID, InitialConfig) -> ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceID)), % % Perform query as further check that the resource is working as expected (fun() -> - erlang:display({pool_name, ResourceID}), QueryNoParamsResWrapper = emqx_resource:query(ResourceID, test_query_no_params()), ?assertMatch({ok, _}, QueryNoParamsResWrapper), {_, QueryNoParamsRes} = QueryNoParamsResWrapper, diff --git a/changes/ee/refactor-10654.en.md b/changes/ee/refactor-10654.en.md new file mode 100644 index 000000000..229d002da --- /dev/null +++ b/changes/ee/refactor-10654.en.md @@ -0,0 +1 @@ +The clickhouse bridge has been refactored so it is located in its own OTP application. diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index de5d8c3b1..7cbc30567 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -4,4 +4,3 @@ mongo_rs_sharded mysql redis redis_cluster -clickhouse diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 6e2dbcbce..ff2f8fed4 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -17,7 +17,8 @@ emqx_bridge_rocketmq, emqx_bridge_rabbitmq, emqx_bridge_tdengine, - emqx_bridge_influxdb + emqx_bridge_influxdb, + emqx_bridge_clickhouse ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 17ffe9b9b..853247c1a 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -32,9 +32,9 @@ api_schemas(Method) -> ref(emqx_bridge_timescale, Method), ref(emqx_bridge_matrix, Method), ref(emqx_bridge_tdengine, Method), - ref(emqx_ee_bridge_clickhouse, Method), ref(emqx_bridge_dynamo, Method), ref(emqx_bridge_rocketmq, Method), + ref(emqx_bridge_clickhouse, Method), ref(emqx_bridge_sqlserver, Method), ref(emqx_bridge_opents, Method), ref(emqx_bridge_pulsar, Method ++ "_producer"), @@ -57,7 +57,7 @@ schema_modules() -> emqx_bridge_timescale, emqx_bridge_matrix, emqx_bridge_tdengine, - emqx_ee_bridge_clickhouse, + emqx_bridge_clickhouse, emqx_bridge_dynamo, emqx_bridge_rocketmq, emqx_bridge_sqlserver, @@ -101,7 +101,7 @@ resource_type(pgsql) -> emqx_connector_pgsql; resource_type(timescale) -> emqx_connector_pgsql; resource_type(matrix) -> emqx_connector_pgsql; resource_type(tdengine) -> emqx_bridge_tdengine_connector; -resource_type(clickhouse) -> emqx_ee_connector_clickhouse; +resource_type(clickhouse) -> emqx_bridge_clickhouse_connector; resource_type(dynamo) -> emqx_bridge_dynamo_connector; resource_type(rocketmq) -> emqx_bridge_rocketmq_connector; resource_type(sqlserver) -> emqx_bridge_sqlserver_connector; @@ -297,7 +297,7 @@ clickhouse_structs() -> [ {clickhouse, mk( - hoconsc:map(name, ref(emqx_ee_bridge_clickhouse, "config")), + hoconsc:map(name, ref(emqx_bridge_clickhouse, "config")), #{ desc => <<"Clickhouse Bridge Config">>, required => false diff --git a/lib-ee/emqx_ee_connector/docker-ct b/lib-ee/emqx_ee_connector/docker-ct index 3db090939..ef579c036 100644 --- a/lib-ee/emqx_ee_connector/docker-ct +++ b/lib-ee/emqx_ee_connector/docker-ct @@ -1,3 +1,2 @@ toxiproxy influxdb -clickhouse diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 3414c80b5..ee1d4e500 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -2,7 +2,6 @@ {erl_opts, [debug_info]}. {deps, [ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, - {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, {emqx, {path, "../../apps/emqx"}}, {emqx_utils, {path, "../../apps/emqx_utils"}} ]}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 9a4f36cf3..e08413968 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -6,8 +6,7 @@ kernel, stdlib, ecpool, - hstreamdb_erl, - clickhouse + hstreamdb_erl ]}, {env, []}, {modules, []}, diff --git a/mix.exs b/mix.exs index 7b76bdc4b..28fa2c6cc 100644 --- a/mix.exs +++ b/mix.exs @@ -158,7 +158,6 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_gcp_pubsub, :emqx_bridge_cassandra, :emqx_bridge_opents, - :emqx_bridge_clickhouse, :emqx_bridge_dynamo, :emqx_bridge_hstreamdb, :emqx_bridge_influxdb, @@ -175,7 +174,8 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_pulsar, :emqx_oracle, :emqx_bridge_oracle, - :emqx_bridge_rabbitmq + :emqx_bridge_rabbitmq, + :emqx_bridge_clickhouse ]) end diff --git a/rel/i18n/emqx_ee_bridge_clickhouse.hocon b/rel/i18n/emqx_bridge_clickhouse.hocon similarity index 98% rename from rel/i18n/emqx_ee_bridge_clickhouse.hocon rename to rel/i18n/emqx_bridge_clickhouse.hocon index 6735aee22..726d1eb7c 100644 --- a/rel/i18n/emqx_ee_bridge_clickhouse.hocon +++ b/rel/i18n/emqx_bridge_clickhouse.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_clickhouse { +emqx_bridge_clickhouse { batch_value_separator.desc: """The default value ',' works for the VALUES format. You can also use other separator if other format is specified. See [INSERT INTO Statement](https://clickhouse.com/docs/en/sql-reference/statements/insert-into).""" diff --git a/rel/i18n/emqx_ee_connector_clickhouse.hocon b/rel/i18n/emqx_bridge_clickhouse_connector.hocon similarity index 89% rename from rel/i18n/emqx_ee_connector_clickhouse.hocon rename to rel/i18n/emqx_bridge_clickhouse_connector.hocon index cebba5aef..26746a52d 100644 --- a/rel/i18n/emqx_ee_connector_clickhouse.hocon +++ b/rel/i18n/emqx_bridge_clickhouse_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_clickhouse { +emqx_bridge_clickhouse_connector { base_url.desc: """The HTTP URL to the Clickhouse server that you want to connect to (for example http://myhostname:8123)""" diff --git a/rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon b/rel/i18n/zh/emqx_bridge_clickhouse.hocon similarity index 97% rename from rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon rename to rel/i18n/zh/emqx_bridge_clickhouse.hocon index a3ede08ba..ff291f667 100644 --- a/rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon +++ b/rel/i18n/zh/emqx_bridge_clickhouse.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_clickhouse { +emqx_bridge_clickhouse { batch_value_separator.desc: """默认为逗号 ','，适用于 VALUE 格式。您也可以使用其他分隔符， 请参考 [INSERT INTO 语句](https://clickhouse.com/docs/en/sql-reference/statements/insert-into)。""" diff --git a/rel/i18n/zh/emqx_ee_connector_clickhouse.hocon b/rel/i18n/zh/emqx_bridge_clickhouse_connector.hocon similarity index 88% rename from rel/i18n/zh/emqx_ee_connector_clickhouse.hocon rename to rel/i18n/zh/emqx_bridge_clickhouse_connector.hocon index f1457a1f6..b6fd7f3b7 100644 --- a/rel/i18n/zh/emqx_ee_connector_clickhouse.hocon +++ b/rel/i18n/zh/emqx_bridge_clickhouse_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_clickhouse { +emqx_bridge_clickhouse_connector { base_url.desc: """你想连接到的Clickhouse服务器的HTTP URL（例如http://myhostname:8123）。""" From cdf42760fac7a35f510a82623e3cf0141c12c5b3 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 17 May 2023 11:01:03 +0200 Subject: [PATCH 117/144] chore: little change avoid atom leak --- apps/emqx/src/emqx_shared_sub.erl | 29 ++++++++++++------------ apps/emqx_authn/src/emqx_authn_api.erl | 4 ++-- apps/emqx_bridge/src/emqx_bridge_api.erl | 6 ++++- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 194e99999..bc13a06c6 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -158,20 +158,18 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) -> -spec strategy(emqx_topic:group()) -> strategy(). strategy(Group) -> - case - emqx:get_config( - [ - broker, - shared_subscription_group, - %%binary_to_existing_atom(Group, utf8), - binary_to_atom(Group), - strategy - ], - undefined - ) - of - undefined -> emqx:get_config([broker, shared_subscription_strategy]); - Strategy -> Strategy + try + emqx:get_config([ + broker, + shared_subscription_group, + binary_to_existing_atom(Group), + strategy + ]) + catch + error:{config_not_found, _} -> + get_default_shared_subscription_strategy(); + error:badarg -> + get_default_shared_subscription_strategy() end. -spec ack_enabled() -> boolean(). @@ -555,3 +553,6 @@ delete_route_if_needed({Group, Topic} = GroupTopic) -> if_no_more_subscribers(GroupTopic, fun() -> ok = emqx_router:do_delete_route(Topic, {Group, node()}) end). + +get_default_shared_subscription_strategy() -> + emqx:get_config([broker, shared_subscription_strategy]). diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 8ce3eadf2..f00ca8ed1 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -794,8 +794,8 @@ find_authenticator_config(AuthenticatorID, ConfKeyPath) -> with_listener(ListenerID, Fun) -> case find_listener(ListenerID) of {ok, {BType, BName}} -> - Type = binary_to_existing_atom(BType, utf8), - Name = binary_to_existing_atom(BName, utf8), + Type = binary_to_existing_atom(BType), + Name = binary_to_existing_atom(BName), ChainName = binary_to_atom(ListenerID), Fun(Type, Name, ChainName); {error, Reason} -> diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index e7a1282ce..7a6eb5120 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -687,11 +687,15 @@ get_metrics_from_local_node(BridgeType, BridgeName) -> ). is_enabled_bridge(BridgeType, BridgeName) -> - try emqx:get_config([bridges, BridgeType, binary_to_atom(BridgeName)]) of + try emqx:get_config([bridges, BridgeType, binary_to_existing_atom(BridgeName)]) of ConfMap -> maps:get(enable, ConfMap, false) catch error:{config_not_found, _} -> + throw(not_found); + error:badarg -> + %% catch non-existing atom, + %% none-existing atom means it is not available in config PT storage. throw(not_found) end. From 01cf7fc70343432a31e46756c276f91f0dd23a1b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 12 May 2023 13:32:13 -0300 Subject: [PATCH 118/144] feat(webhook): use clientid for hash-loadbalancing ehttpc workers --- apps/emqx_connector/src/emqx_connector_http.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index bb822a60a..02573fc41 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -268,9 +268,10 @@ on_query(InstId, {send_message, Msg}, State) -> } = process_request(Request, Msg), %% bridge buffer worker has retry, do not let ehttpc retry Retry = 0, + ClientId = maps:get(clientid, Msg, undefined), on_query( InstId, - {undefined, Method, {Path, Headers, Body}, Timeout, Retry}, + {ClientId, Method, {Path, Headers, Body}, Timeout, Retry}, State ) end; @@ -370,9 +371,10 @@ on_query_async(InstId, {send_message, Msg}, ReplyFunAndArgs, State) -> headers := Headers, request_timeout := Timeout } = process_request(Request, Msg), + ClientId = maps:get(clientid, Msg, undefined), on_query_async( InstId, - {undefined, Method, {Path, Headers, Body}, Timeout}, + {ClientId, Method, {Path, Headers, Body}, Timeout}, ReplyFunAndArgs, State ) From a7b41e1cdff1be3179a5b7dc461fd678d10e1d61 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 12 May 2023 13:04:27 -0300 Subject: [PATCH 119/144] perf(webhook): add retry attempts for async This is a performance improvement for webhook bridge. Since this bridge is called using `async` callback mode, and `ehttpc` frequently returns errors of the form `normal` and `{shutdown, normal}` that are retried "for free" by `ehttpc`, we add this behavior to async requests as well. Other errors are retried too, but they are not "free": 3 attempts are made at a maximum. This is important because, when using buffer workers, we should avoid making them enter the `blocked` state, since that halts all progress and makes throughput plummet. --- .../test/emqx_bridge_webhook_SUITE.erl | 110 +++++++++++++++++- .../src/emqx_connector_http.erl | 62 ++++++++-- changes/ce/perf-10690.en.md | 3 + 3 files changed, 162 insertions(+), 13 deletions(-) create mode 100644 changes/ce/perf-10690.en.md diff --git a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl index f08c87b6e..45cc82251 100644 --- a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl @@ -23,6 +23,7 @@ -compile(export_all). -import(emqx_mgmt_api_test_util, [request/3, uri/1]). +-import(emqx_common_test_helpers, [on_exit/1]). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -52,6 +53,13 @@ end_per_suite(_Config) -> suite() -> [{timetrap, {seconds, 60}}]. +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + emqx_common_test_helpers:call_janitor(), + ok. + %%------------------------------------------------------------------------------ %% HTTP server for testing %% (Orginally copied from emqx_bridge_api_SUITE) @@ -158,7 +166,8 @@ bridge_async_config(#{port := Port} = Config) -> QueryMode = maps:get(query_mode, Config, "async"), ConnectTimeout = maps:get(connect_timeout, Config, 1), RequestTimeout = maps:get(request_timeout, Config, 10000), - ResourceRequestTimeout = maps:get(resouce_request_timeout, Config, "infinity"), + ResumeInterval = maps:get(resume_interval, Config, "1s"), + ResourceRequestTimeout = maps:get(resource_request_timeout, Config, "infinity"), ConfigString = io_lib:format( "bridges.~s.~s {\n" " url = \"http://localhost:~p\"\n" @@ -177,7 +186,8 @@ bridge_async_config(#{port := Port} = Config) -> " health_check_interval = \"15s\"\n" " max_buffer_bytes = \"1GB\"\n" " query_mode = \"~s\"\n" - " request_timeout = \"~s\"\n" + " request_timeout = \"~p\"\n" + " resume_interval = \"~s\"\n" " start_after_created = \"true\"\n" " start_timeout = \"5s\"\n" " worker_pool_size = \"1\"\n" @@ -194,7 +204,8 @@ bridge_async_config(#{port := Port} = Config) -> PoolSize, RequestTimeout, QueryMode, - ResourceRequestTimeout + ResourceRequestTimeout, + ResumeInterval ] ), ct:pal(ConfigString), @@ -236,7 +247,7 @@ t_send_async_connection_timeout(_Config) -> query_mode => "async", connect_timeout => ResponseDelayMS * 2, request_timeout => 10000, - resouce_request_timeout => "infinity" + resource_request_timeout => "infinity" }), NumberOfMessagesToSend = 10, [ @@ -250,6 +261,97 @@ t_send_async_connection_timeout(_Config) -> stop_http_server(Server), ok. +t_async_free_retries(_Config) -> + #{port := Port} = start_http_server(#{response_delay_ms => 0}), + BridgeID = make_bridge(#{ + port => Port, + pool_size => 1, + query_mode => "sync", + connect_timeout => 1_000, + request_timeout => 10_000, + resource_request_timeout => "10000s" + }), + %% Fail 5 times then succeed. + Context = #{error_attempts => 5}, + ExpectedAttempts = 6, + Fn = fun(Get, Error) -> + ?assertMatch( + {ok, 200, _, _}, + emqx_bridge:send_message(BridgeID, #{<<"hello">> => <<"world">>}), + #{error => Error} + ), + ?assertEqual(ExpectedAttempts, Get(), #{error => Error}) + end, + do_t_async_retries(Context, {error, normal}, Fn), + do_t_async_retries(Context, {error, {shutdown, normal}}, Fn), + ok. + +t_async_common_retries(_Config) -> + #{port := Port} = start_http_server(#{response_delay_ms => 0}), + BridgeID = make_bridge(#{ + port => Port, + pool_size => 1, + query_mode => "sync", + resume_interval => "100ms", + connect_timeout => 1_000, + request_timeout => 10_000, + resource_request_timeout => "10000s" + }), + %% Keeps failing until connector gives up. + Context = #{error_attempts => infinity}, + ExpectedAttempts = 3, + FnSucceed = fun(Get, Error) -> + ?assertMatch( + {ok, 200, _, _}, + emqx_bridge:send_message(BridgeID, #{<<"hello">> => <<"world">>}), + #{error => Error, attempts => Get()} + ), + ?assertEqual(ExpectedAttempts, Get(), #{error => Error}) + end, + FnFail = fun(Get, Error) -> + ?assertMatch( + Error, + emqx_bridge:send_message(BridgeID, #{<<"hello">> => <<"world">>}), + #{error => Error, attempts => Get()} + ), + ?assertEqual(ExpectedAttempts, Get(), #{error => Error}) + end, + %% These two succeed because they're further retried by the buffer + %% worker synchronously, and we're not mock that call. + do_t_async_retries(Context, {error, {closed, "The connection was lost."}}, FnSucceed), + do_t_async_retries(Context, {error, {shutdown, closed}}, FnSucceed), + %% This fails because this error is treated as unrecoverable. + do_t_async_retries(Context, {error, something_else}, FnFail), + ok. + +do_t_async_retries(TestContext, Error, Fn) -> + #{error_attempts := ErrorAttempts} = TestContext, + persistent_term:put({?MODULE, ?FUNCTION_NAME, attempts}, 0), + on_exit(fun() -> persistent_term:erase({?MODULE, ?FUNCTION_NAME, attempts}) end), + Get = fun() -> persistent_term:get({?MODULE, ?FUNCTION_NAME, attempts}) end, + GetAndBump = fun() -> + Attempts = persistent_term:get({?MODULE, ?FUNCTION_NAME, attempts}), + persistent_term:put({?MODULE, ?FUNCTION_NAME, attempts}, Attempts + 1), + Attempts + 1 + end, + emqx_common_test_helpers:with_mock( + emqx_connector_http, + reply_delegator, + fun(Context, ReplyFunAndArgs, Result) -> + Attempts = GetAndBump(), + case Attempts > ErrorAttempts of + true -> + ct:pal("succeeding ~p : ~p", [Error, Attempts]), + meck:passthrough([Context, ReplyFunAndArgs, Result]); + false -> + ct:pal("failing ~p : ~p", [Error, Attempts]), + meck:passthrough([Context, ReplyFunAndArgs, Error]) + end + end, + fun() -> Fn(Get, Error) end + ), + ok. + receive_request_notifications(MessageIDs, _ResponseDelay) when map_size(MessageIDs) =:= 0 -> ok; receive_request_notifications(MessageIDs, ResponseDelay) -> diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 02573fc41..275b933f7 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -32,7 +32,7 @@ on_query/3, on_query_async/4, on_get_status/2, - reply_delegator/2 + reply_delegator/3 ]). -type url() :: emqx_http_lib:uri_map(). @@ -267,7 +267,7 @@ on_query(InstId, {send_message, Msg}, State) -> request_timeout := Timeout } = process_request(Request, Msg), %% bridge buffer worker has retry, do not let ehttpc retry - Retry = 0, + Retry = 2, ClientId = maps:get(clientid, Msg, undefined), on_query( InstId, @@ -396,12 +396,22 @@ on_query_async( } ), NRequest = formalize_request(Method, BasePath, Request), + MaxAttempts = maps:get(max_attempts, State, 3), + Context = #{ + attempt => 1, + max_attempts => MaxAttempts, + state => State, + key_or_num => KeyOrNum, + method => Method, + request => NRequest, + timeout => Timeout + }, ok = ehttpc:request_async( Worker, Method, NRequest, Timeout, - {fun ?MODULE:reply_delegator/2, [ReplyFunAndArgs]} + {fun ?MODULE:reply_delegator/3, [Context, ReplyFunAndArgs]} ), {ok, Worker}. @@ -634,7 +644,10 @@ to_bin(Str) when is_list(Str) -> to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). -reply_delegator(ReplyFunAndArgs, Result) -> +reply_delegator(Context, ReplyFunAndArgs, Result) -> + maybe_retry(Result, Context, ReplyFunAndArgs). + +transform_result(Result) -> case Result of %% The normal reason happens when the HTTP connection times out before %% the request has been fully processed @@ -645,16 +658,47 @@ reply_delegator(ReplyFunAndArgs, Result) -> Reason =:= {shutdown, normal}; Reason =:= {shutdown, closed} -> - Result1 = {error, {recoverable_error, Reason}}, - emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result1); + {error, {recoverable_error, Reason}}; {error, {closed, _Message} = Reason} -> %% _Message = "The connection was lost." - Result1 = {error, {recoverable_error, Reason}}, - emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result1); + {error, {recoverable_error, Reason}}; _ -> - emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result) + Result end. +maybe_retry(Result0, _Context = #{attempt := N, max_attempts := Max}, ReplyFunAndArgs) when + N >= Max +-> + Result = transform_result(Result0), + emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result); +maybe_retry({error, Reason}, Context, ReplyFunAndArgs) -> + #{ + state := State, + attempt := Attempt, + key_or_num := KeyOrNum, + method := Method, + request := Request, + timeout := Timeout + } = Context, + %% TODO: reset the expiration time for free retries? + IsFreeRetry = Reason =:= normal orelse Reason =:= {shutdown, normal}, + NContext = + case IsFreeRetry of + true -> Context; + false -> Context#{attempt := Attempt + 1} + end, + Worker = resolve_pool_worker(State, KeyOrNum), + ok = ehttpc:request_async( + Worker, + Method, + Request, + Timeout, + {fun ?MODULE:reply_delegator/3, [NContext, ReplyFunAndArgs]} + ), + ok; +maybe_retry(Result, _Context, ReplyFunAndArgs) -> + emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result). + %% The HOCON schema system may generate sensitive keys with this format is_sensitive_key([{str, StringKey}]) -> is_sensitive_key(StringKey); diff --git a/changes/ce/perf-10690.en.md b/changes/ce/perf-10690.en.md new file mode 100644 index 000000000..4f7c49e53 --- /dev/null +++ b/changes/ce/perf-10690.en.md @@ -0,0 +1,3 @@ +Added a retry mechanism to webhook bridge that attempts to improve throughput. + +This optimization retries request failures without blocking the buffering layer, which can improve throughput in situations of high messaging rate. From bc7d0d5fd62a9c9f18932012713309e8b1e33531 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 12 May 2023 13:52:11 -0300 Subject: [PATCH 120/144] perf(webhook): evaluate replies in fresh process rather than in ehttpc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This surprisingly simple change yields a big performance improvement in throughput. While the previous commit achieves ~ 55 k messages / s in throughput under some test conditions (100 k concurrent publishers publishing 1 QoS 1 message per second), the simple change in this commit improves it further to ~ 63 k messages / s. Benchmarks indicated that the evaluating one reply function is consistently quite fast (~ 20 µs), which makes this performance gain counterintuitive. Perhaps, although each call is cheap, `ehttpc` calls several of these in a row when there are several sent requests, and those costs might add up in latency. --- apps/emqx_connector/src/emqx_connector_http.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 275b933f7..4891c9595 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -645,7 +645,7 @@ to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). reply_delegator(Context, ReplyFunAndArgs, Result) -> - maybe_retry(Result, Context, ReplyFunAndArgs). + spawn(fun() -> maybe_retry(Result, Context, ReplyFunAndArgs) end). transform_result(Result) -> case Result of From ea4fbb8b6e2524abd1d031996438625158978a6a Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 17 May 2023 14:46:07 +0200 Subject: [PATCH 121/144] chore: add changelog --- changes/ce/perf-10698.en.md | 1 + changes/ce/perf-10698.zh.md | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 changes/ce/perf-10698.en.md create mode 100644 changes/ce/perf-10698.zh.md diff --git a/changes/ce/perf-10698.en.md b/changes/ce/perf-10698.en.md new file mode 100644 index 000000000..db398b919 --- /dev/null +++ b/changes/ce/perf-10698.en.md @@ -0,0 +1 @@ +Optimize memory usage when accessing the configuration during runtime. diff --git a/changes/ce/perf-10698.zh.md b/changes/ce/perf-10698.zh.md new file mode 100644 index 000000000..ea7f88259 --- /dev/null +++ b/changes/ce/perf-10698.zh.md @@ -0,0 +1,5 @@ +在运行时降低读取配置的内存占用。 + + + + From e26ce5816e28c60cc150e88579c8707f7ecb90c3 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 15 May 2023 14:42:19 +0800 Subject: [PATCH 122/144] feat: hide keepalive_backoff, introduce keepalive_multiplier --- apps/emqx/src/emqx_channel.erl | 11 ++- apps/emqx/src/emqx_keepalive.erl | 94 +++++++------------ apps/emqx/src/emqx_schema.erl | 46 ++++++++- apps/emqx/test/emqx_keepalive_SUITE.erl | 10 +- apps/emqx/test/emqx_schema_tests.erl | 37 ++++++++ .../src/emqx_mgmt_api_clients.erl | 3 +- .../test/emqx_mgmt_api_clients_SUITE.erl | 22 ++++- changes/ce/feat-10702.en.md | 4 + rel/i18n/emqx_schema.hocon | 11 ++- 9 files changed, 156 insertions(+), 82 deletions(-) create mode 100644 changes/ce/feat-10702.en.md diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 3fb6a5f6b..3c35f0ff4 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1199,12 +1199,13 @@ handle_call(list_authz_cache, Channel) -> handle_call( {keepalive, Interval}, Channel = #channel{ - keepalive = KeepAlive, + keepalive = _KeepAlive, conninfo = ConnInfo } ) -> ClientId = info(clientid, Channel), - NKeepalive = emqx_keepalive:set(interval, Interval * 1000, KeepAlive), + RecvCnt = emqx_pd:get_counter(recv_pkt), + NKeepalive = emqx_keepalive:init(RecvCnt, Interval * 1000), NConnInfo = maps:put(keepalive, Interval, ConnInfo), NChannel = Channel#channel{keepalive = NKeepalive, conninfo = NConnInfo}, SockInfo = maps:get(sockinfo, emqx_cm:get_chan_info(ClientId), #{}), @@ -2025,9 +2026,9 @@ ensure_keepalive_timer(0, Channel) -> ensure_keepalive_timer(disabled, Channel) -> Channel; ensure_keepalive_timer(Interval, Channel = #channel{clientinfo = #{zone := Zone}}) -> - Backoff = get_mqtt_conf(Zone, keepalive_backoff), - RecvOct = emqx_pd:get_counter(incoming_bytes), - Keepalive = emqx_keepalive:init(RecvOct, round(timer:seconds(Interval) * Backoff)), + Multiplier = get_mqtt_conf(Zone, keepalive_multiplier), + RecvCnt = emqx_pd:get_counter(recv_pkt), + Keepalive = emqx_keepalive:init(RecvCnt, round(timer:seconds(Interval) * Multiplier)), ensure_timer(alive_timer, Channel#channel{keepalive = Keepalive}). clear_keepalive(Channel = #channel{timers = Timers}) -> diff --git a/apps/emqx/src/emqx_keepalive.erl b/apps/emqx/src/emqx_keepalive.erl index 9ba11e23f..2333760ba 100644 --- a/apps/emqx/src/emqx_keepalive.erl +++ b/apps/emqx/src/emqx_keepalive.erl @@ -21,8 +21,7 @@ init/2, info/1, info/2, - check/2, - set/3 + check/2 ]). -elvis([{elvis_style, no_if_expression, disable}]). @@ -31,66 +30,16 @@ -record(keepalive, { interval :: pos_integer(), - statval :: non_neg_integer(), - repeat :: non_neg_integer() + statval :: non_neg_integer() }). -opaque keepalive() :: #keepalive{}. +-define(MAX_INTERVAL, 65535000). %% @doc Init keepalive. -spec init(Interval :: non_neg_integer()) -> keepalive(). init(Interval) -> init(0, Interval). -%% @doc Init keepalive. --spec init(StatVal :: non_neg_integer(), Interval :: non_neg_integer()) -> keepalive(). -init(StatVal, Interval) when Interval > 0 -> - #keepalive{ - interval = Interval, - statval = StatVal, - repeat = 0 - }. - -%% @doc Get Info of the keepalive. --spec info(keepalive()) -> emqx_types:infos(). -info(#keepalive{ - interval = Interval, - statval = StatVal, - repeat = Repeat -}) -> - #{ - interval => Interval, - statval => StatVal, - repeat => Repeat - }. - --spec info(interval | statval | repeat, keepalive()) -> - non_neg_integer(). -info(interval, #keepalive{interval = Interval}) -> - Interval; -info(statval, #keepalive{statval = StatVal}) -> - StatVal; -info(repeat, #keepalive{repeat = Repeat}) -> - Repeat. - -%% @doc Check keepalive. --spec check(non_neg_integer(), keepalive()) -> - {ok, keepalive()} | {error, timeout}. -check( - NewVal, - KeepAlive = #keepalive{ - statval = OldVal, - repeat = Repeat - } -) -> - if - NewVal =/= OldVal -> - {ok, KeepAlive#keepalive{statval = NewVal, repeat = 0}}; - Repeat < 1 -> - {ok, KeepAlive#keepalive{repeat = Repeat + 1}}; - true -> - {error, timeout} - end. - %% from mqtt-v3.1.1 specific %% A Keep Alive value of zero (0) has the effect of turning off the keep alive mechanism. %% This means that, in this case, the Server is not required @@ -102,7 +51,36 @@ check( %%The actual value of the Keep Alive is application specific; %% typically this is a few minutes. %% The maximum value is (65535s) 18 hours 12 minutes and 15 seconds. -%% @doc Update keepalive's interval --spec set(interval, non_neg_integer(), keepalive()) -> keepalive(). -set(interval, Interval, KeepAlive) when Interval >= 0 andalso Interval =< 65535000 -> - KeepAlive#keepalive{interval = Interval}. +%% @doc Init keepalive. +-spec init(StatVal :: non_neg_integer(), Interval :: non_neg_integer()) -> keepalive(). +init(StatVal, Interval) when Interval > 0 andalso Interval =< ?MAX_INTERVAL -> + #keepalive{interval = Interval, statval = StatVal}; +init(_, 0) -> + undefined; +init(StatVal, Interval) when Interval > ?MAX_INTERVAL -> init(StatVal, ?MAX_INTERVAL). + +%% @doc Get Info of the keepalive. +-spec info(keepalive()) -> emqx_types:infos(). +info(#keepalive{ + interval = Interval, + statval = StatVal +}) -> + #{ + interval => Interval, + statval => StatVal + }. + +-spec info(interval | statval, keepalive()) -> + non_neg_integer(). +info(interval, #keepalive{interval = Interval}) -> + Interval; +info(statval, #keepalive{statval = StatVal}) -> + StatVal; +info(interval, undefined) -> + 0. + +%% @doc Check keepalive. +-spec check(non_neg_integer(), keepalive()) -> + {ok, keepalive()} | {error, timeout}. +check(Val, #keepalive{statval = Val}) -> {error, timeout}; +check(Val, KeepAlive) -> {ok, KeepAlive#keepalive{statval = Val}}. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 5a66ad5a0..0e9a29f3d 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -77,6 +77,7 @@ validate_heap_size/1, user_lookup_fun_tr/2, validate_alarm_actions/1, + validate_keepalive_multiplier/1, non_empty_string/1, validations/0, naive_env_interpolation/1 @@ -109,7 +110,8 @@ servers_validator/2, servers_sc/2, convert_servers/1, - convert_servers/2 + convert_servers/2, + mqtt_converter/2 ]). %% tombstone types @@ -150,6 +152,8 @@ -define(BIT(Bits), (1 bsl (Bits))). -define(MAX_UINT(Bits), (?BIT(Bits) - 1)). +-define(DEFAULT_MULTIPLIER, 1.5). +-define(DEFAULT_BACKOFF, 0.75). namespace() -> broker. @@ -172,6 +176,7 @@ roots(high) -> ref("mqtt"), #{ desc => ?DESC(mqtt), + converter => fun ?MODULE:mqtt_converter/2, importance => ?IMPORTANCE_MEDIUM } )}, @@ -522,8 +527,19 @@ fields("mqtt") -> sc( number(), #{ - default => 0.75, - desc => ?DESC(mqtt_keepalive_backoff) + default => ?DEFAULT_BACKOFF, + %% Must add required => false, zone schema has no default. + required => false, + importance => ?IMPORTANCE_HIDDEN + } + )}, + {"keepalive_multiplier", + sc( + number(), + #{ + default => ?DEFAULT_MULTIPLIER, + validator => fun ?MODULE:validate_keepalive_multiplier/1, + desc => ?DESC(mqtt_keepalive_multiplier) } )}, {"max_subscriptions", @@ -2744,6 +2760,13 @@ validate_heap_size(Siz) when is_integer(Siz) -> validate_heap_size(_SizStr) -> {error, invalid_heap_size}. +validate_keepalive_multiplier(Multiplier) when + is_number(Multiplier) andalso Multiplier >= 1.0 andalso Multiplier =< 65535.0 +-> + ok; +validate_keepalive_multiplier(_Multiplier) -> + {error, #{reason => keepalive_multiplier_out_of_range, min => 1, max => 65535}}. + validate_alarm_actions(Actions) -> UnSupported = lists:filter( fun(Action) -> Action =/= log andalso Action =/= publish end, Actions @@ -3385,3 +3408,20 @@ ensure_default_listener(Map, ListenerType) -> cert_file(_File, client) -> undefined; cert_file(File, server) -> iolist_to_binary(filename:join(["${EMQX_ETC_DIR}", "certs", File])). + +mqtt_converter(#{<<"keepalive_multiplier">> := Multi} = Mqtt, _Opts) -> + case round(Multi * 100) =:= round(?DEFAULT_MULTIPLIER * 100) of + false -> + %% Multiplier is provided, and it's not default value + Mqtt; + true -> + %% Multiplier is default value, fallback to use Backoff value + %% Backoff default value was half of Multiplier default value + %% so there is no need to compare Backoff with its default. + Backoff = maps:get(<<"keepalive_backoff">>, Mqtt, ?DEFAULT_BACKOFF), + Mqtt#{<<"keepalive_multiplier">> => Backoff * 2} + end; +mqtt_converter(#{<<"keepalive_backoff">> := Backoff} = Mqtt, _Opts) -> + Mqtt#{<<"keepalive_multiplier">> => Backoff * 2}; +mqtt_converter(Mqtt, _Opts) -> + Mqtt. diff --git a/apps/emqx/test/emqx_keepalive_SUITE.erl b/apps/emqx/test/emqx_keepalive_SUITE.erl index dce55409e..480beeaa4 100644 --- a/apps/emqx/test/emqx_keepalive_SUITE.erl +++ b/apps/emqx/test/emqx_keepalive_SUITE.erl @@ -27,20 +27,14 @@ t_check(_) -> Keepalive = emqx_keepalive:init(60), ?assertEqual(60, emqx_keepalive:info(interval, Keepalive)), ?assertEqual(0, emqx_keepalive:info(statval, Keepalive)), - ?assertEqual(0, emqx_keepalive:info(repeat, Keepalive)), Info = emqx_keepalive:info(Keepalive), ?assertEqual( #{ interval => 60, - statval => 0, - repeat => 0 + statval => 0 }, Info ), {ok, Keepalive1} = emqx_keepalive:check(1, Keepalive), ?assertEqual(1, emqx_keepalive:info(statval, Keepalive1)), - ?assertEqual(0, emqx_keepalive:info(repeat, Keepalive1)), - {ok, Keepalive2} = emqx_keepalive:check(1, Keepalive1), - ?assertEqual(1, emqx_keepalive:info(statval, Keepalive2)), - ?assertEqual(1, emqx_keepalive:info(repeat, Keepalive2)), - ?assertEqual({error, timeout}, emqx_keepalive:check(1, Keepalive2)). + ?assertEqual({error, timeout}, emqx_keepalive:check(1, Keepalive1)). diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index 81991f26e..3dcfa331e 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -655,6 +655,43 @@ password_converter_test() -> ?assertThrow("must_quote", emqx_schema:password_converter(foobar, #{})), ok. +-define(MQTT(B, M), #{<<"keepalive_backoff">> => B, <<"keepalive_multiplier">> => M}). + +keepalive_convert_test() -> + ?assertEqual(undefined, emqx_schema:mqtt_converter(undefined, #{})), + DefaultBackoff = 0.75, + DefaultMultiplier = 1.5, + Default = ?MQTT(DefaultBackoff, DefaultMultiplier), + ?assertEqual(Default, emqx_schema:mqtt_converter(Default, #{})), + ?assertEqual(?MQTT(1.5, 3), emqx_schema:mqtt_converter(?MQTT(1.5, 3), #{})), + ?assertEqual( + ?MQTT(DefaultBackoff, 3), emqx_schema:mqtt_converter(?MQTT(DefaultBackoff, 3), #{}) + ), + ?assertEqual(?MQTT(1, 2), emqx_schema:mqtt_converter(?MQTT(1, DefaultMultiplier), #{})), + ?assertEqual(?MQTT(1.5, 3), emqx_schema:mqtt_converter(?MQTT(1.5, 3), #{})), + + ?assertEqual(#{}, emqx_schema:mqtt_converter(#{}, #{})), + ?assertEqual( + #{<<"keepalive_backoff">> => 1.5, <<"keepalive_multiplier">> => 3.0}, + emqx_schema:mqtt_converter(#{<<"keepalive_backoff">> => 1.5}, #{}) + ), + ?assertEqual( + #{<<"keepalive_multiplier">> => 5.0}, + emqx_schema:mqtt_converter(#{<<"keepalive_multiplier">> => 5.0}, #{}) + ), + ?assertEqual( + #{ + <<"keepalive_backoff">> => DefaultBackoff, + <<"keepalive_multiplier">> => DefaultMultiplier + }, + emqx_schema:mqtt_converter(#{<<"keepalive_backoff">> => DefaultBackoff}, #{}) + ), + ?assertEqual( + #{<<"keepalive_multiplier">> => DefaultMultiplier}, + emqx_schema:mqtt_converter(#{<<"keepalive_multiplier">> => DefaultMultiplier}, #{}) + ), + ok. + url_type_test_() -> [ ?_assertEqual( diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 681c851bf..846a1d466 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -546,7 +546,8 @@ fields(authz_cache) -> ]; fields(keepalive) -> [ - {interval, hoconsc:mk(integer(), #{desc => <<"Keepalive time, with the unit of second">>})} + {interval, + hoconsc:mk(range(0, 65535), #{desc => <<"Keepalive time, with the unit of second">>})} ]; fields(subscribe) -> [ diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 6d7733b22..89838c346 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -244,13 +244,31 @@ t_keepalive(_Config) -> Body = #{interval => 11}, {error, {"HTTP/1.1", 404, "Not Found"}} = emqx_mgmt_api_test_util:request_api(put, Path, <<"">>, AuthHeader, Body), - {ok, C1} = emqtt:start_link(#{username => Username, clientid => ClientId}), + %% 65535 is the max value of keepalive + MaxKeepalive = 65535, + InitKeepalive = round(MaxKeepalive / 1.5 + 1), + {ok, C1} = emqtt:start_link(#{ + username => Username, clientid => ClientId, keepalive => InitKeepalive + }), {ok, _} = emqtt:connect(C1), + [Pid] = emqx_cm:lookup_channels(list_to_binary(ClientId)), + %% will reset to max keepalive if keepalive > max keepalive + #{conninfo := #{keepalive := InitKeepalive}} = emqx_connection:info(Pid), + ?assertMatch({keepalive, 65535000, _}, element(5, element(9, sys:get_state(Pid)))), + {ok, NewClient} = emqx_mgmt_api_test_util:request_api(put, Path, <<"">>, AuthHeader, Body), #{<<"keepalive">> := 11} = emqx_utils_json:decode(NewClient, [return_maps]), - [Pid] = emqx_cm:lookup_channels(list_to_binary(ClientId)), #{conninfo := #{keepalive := Keepalive}} = emqx_connection:info(Pid), ?assertEqual(11, Keepalive), + %% Disable keepalive + Body1 = #{interval => 0}, + {ok, NewClient1} = emqx_mgmt_api_test_util:request_api(put, Path, <<"">>, AuthHeader, Body1), + #{<<"keepalive">> := 0} = emqx_utils_json:decode(NewClient1, [return_maps]), + ?assertMatch(#{conninfo := #{keepalive := 0}}, emqx_connection:info(Pid)), + %% Maximal keepalive + Body2 = #{interval => 65536}, + {error, {"HTTP/1.1", 400, _}} = + emqx_mgmt_api_test_util:request_api(put, Path, <<"">>, AuthHeader, Body2), emqtt:disconnect(C1), ok. diff --git a/changes/ce/feat-10702.en.md b/changes/ce/feat-10702.en.md new file mode 100644 index 000000000..936103848 --- /dev/null +++ b/changes/ce/feat-10702.en.md @@ -0,0 +1,4 @@ +Introduce a more straightforward configuration option `keepalive_multiplier` and +deprecate the old `keepalive_backoff` configuration. +After this enhancement, EMQX checks the client's keepalive timeout status +period by multiplying the "Client Requested Keepalive Interval" with `keepalive_multiplier`. diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index ac8b7e8a4..ad63b4ba9 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -799,7 +799,7 @@ fields_tcp_opts_high_watermark.desc: by the VM socket implementation reaches this limit.""" fields_tcp_opts_high_watermark.label: -"""TCP 高水位线""" +"""TCP high watermark""" fields_mqtt_quic_listener_stateless_operation_expiration_ms.desc: """The time limit between operations for the same endpoint, in milliseconds. Default: 100""" @@ -885,11 +885,12 @@ and an MQTT message is published to the system topic $SYS/sysmon/long_sche sysmon_vm_long_schedule.label: """Enable Long Schedule monitoring.""" -mqtt_keepalive_backoff.desc: -"""The coefficient EMQX uses to confirm whether the keep alive duration of the client expires. Formula: Keep Alive * Backoff * 2""" +mqtt_keepalive_multiplier.desc: +"""Keep-Alive Timeout = Keep-Alive interval × Keep-Alive Multiplier. +The default value 1.5 is following the MQTT 5.0 specification. This multiplier is adjustable, providing system administrators flexibility for tailoring to their specific needs. For instance, if a client's 10-second Keep-Alive interval PINGREQ gets delayed by an extra 10 seconds, changing the multiplier to 2 lets EMQX tolerate this delay.""" -mqtt_keepalive_backoff.label: -"""Keep Alive Backoff""" +mqtt_keepalive_multiplier.label: +"""Keep Alive Multiplier""" force_gc_bytes.desc: """GC the process after specified number of bytes have passed through.""" From 727ad5999563fa20eed2f12f573903d54ea8ffa0 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 17 May 2023 16:07:27 +0200 Subject: [PATCH 123/144] chore(test): improve coverage --- apps/emqx/src/emqx.erl | 9 +++++++-- apps/emqx/test/emqx_SUITE.erl | 13 +++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index 53e6b9536..1cdb563aa 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -189,8 +189,13 @@ get_config(KeyPath) -> -spec get_config(emqx_utils_maps:config_key_path(), term()) -> term(). get_config(KeyPath, Default) -> - KeyPath1 = emqx_config:ensure_atom_conf_path(KeyPath, {return, Default}), - emqx_config:get(KeyPath1, Default). + try + KeyPath1 = emqx_config:ensure_atom_conf_path(KeyPath, {raise_error, config_not_found}), + emqx_config:get(KeyPath1, Default) + catch + error:config_not_found -> + Default + end. -spec get_raw_config(emqx_utils_maps:config_key_path()) -> term(). get_raw_config(KeyPath) -> diff --git a/apps/emqx/test/emqx_SUITE.erl b/apps/emqx/test/emqx_SUITE.erl index 64ed2ea19..cfabff401 100644 --- a/apps/emqx/test/emqx_SUITE.erl +++ b/apps/emqx/test/emqx_SUITE.erl @@ -156,6 +156,19 @@ t_cluster_nodes(_) -> ?assertEqual(Expected, emqx:cluster_nodes(cores)), ?assertEqual([], emqx:cluster_nodes(stopped)). +t_get_config(_) -> + ?assertEqual(false, emqx:get_config([overload_protection, enable])), + ?assertEqual(false, emqx:get_config(["overload_protection", <<"enable">>])). + +t_get_config_default_1(_) -> + ?assertEqual(false, emqx:get_config([overload_protection, enable], undefined)), + ?assertEqual(false, emqx:get_config(["overload_protection", <<"enable">>], undefined)). + +t_get_config_default_2(_) -> + AtomPathRes = emqx:get_config([overload_protection, <<"_!no_@exist_">>], undefined), + NonAtomPathRes = emqx:get_config(["doesnotexist", <<"db_backend">>], undefined), + ?assertEqual(undefined, NonAtomPathRes), + ?assertEqual(undefined, AtomPathRes). %%-------------------------------------------------------------------- %% Hook fun %%-------------------------------------------------------------------- From f83f9a2d795c09a466b5c5896c845d599cde76c5 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 17 May 2023 14:56:11 -0300 Subject: [PATCH 124/144] chore: bump app vsns --- apps/emqx_authn/src/emqx_authn.app.src | 2 +- apps/emqx_authz/src/emqx_authz.app.src | 2 +- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_connector/src/emqx_connector.app.src | 2 +- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_management/src/emqx_management.app.src | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src | 2 +- lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index 3e0cf786e..a291f2f61 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.19"}, + {vsn, "0.1.20"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]}, diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index dd0325694..4856008e6 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.19"}, + {vsn, "0.1.20"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index e6c3d9cd9..c31a16b9b 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.19"}, + {vsn, "0.1.20"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index db55c7032..76c3e8bb9 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.22"}, + {vsn, "0.1.23"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 2ffca464d..26dab4c9c 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.16"}, + {vsn, "0.1.17"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, emqx, emqx_authn, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 34f3dd1fe..f51a83923 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.21"}, + {vsn, "5.0.22"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index ff2f8fed4..ae922ecee 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.12"}, + {vsn, "0.1.13"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index e08413968..3ed460492 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_connector, [ {description, "EMQX Enterprise connectors"}, - {vsn, "0.1.12"}, + {vsn, "0.1.13"}, {registered, []}, {applications, [ kernel, From d4d25d26600a217eec4bc1f7afa033aabb11dda7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Thu, 18 May 2023 09:44:01 +0800 Subject: [PATCH 125/144] chore: don't lost previous's statval --- apps/emqx/src/emqx_channel.erl | 5 ++--- apps/emqx/src/emqx_keepalive.erl | 12 ++++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 3c35f0ff4..851afb40e 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1199,13 +1199,12 @@ handle_call(list_authz_cache, Channel) -> handle_call( {keepalive, Interval}, Channel = #channel{ - keepalive = _KeepAlive, + keepalive = KeepAlive, conninfo = ConnInfo } ) -> ClientId = info(clientid, Channel), - RecvCnt = emqx_pd:get_counter(recv_pkt), - NKeepalive = emqx_keepalive:init(RecvCnt, Interval * 1000), + NKeepalive = emqx_keepalive:update(timer:seconds(Interval), KeepAlive), NConnInfo = maps:put(keepalive, Interval, ConnInfo), NChannel = Channel#channel{keepalive = NKeepalive, conninfo = NConnInfo}, SockInfo = maps:get(sockinfo, emqx_cm:get_chan_info(ClientId), #{}), diff --git a/apps/emqx/src/emqx_keepalive.erl b/apps/emqx/src/emqx_keepalive.erl index 2333760ba..c0a1c7657 100644 --- a/apps/emqx/src/emqx_keepalive.erl +++ b/apps/emqx/src/emqx_keepalive.erl @@ -21,7 +21,8 @@ init/2, info/1, info/2, - check/2 + check/2, + update/2 ]). -elvis([{elvis_style, no_if_expression, disable}]). @@ -52,7 +53,7 @@ init(Interval) -> init(0, Interval). %% typically this is a few minutes. %% The maximum value is (65535s) 18 hours 12 minutes and 15 seconds. %% @doc Init keepalive. --spec init(StatVal :: non_neg_integer(), Interval :: non_neg_integer()) -> keepalive(). +-spec init(StatVal :: non_neg_integer(), Interval :: non_neg_integer()) -> keepalive() | undefined. init(StatVal, Interval) when Interval > 0 andalso Interval =< ?MAX_INTERVAL -> #keepalive{interval = Interval, statval = StatVal}; init(_, 0) -> @@ -84,3 +85,10 @@ info(interval, undefined) -> {ok, keepalive()} | {error, timeout}. check(Val, #keepalive{statval = Val}) -> {error, timeout}; check(Val, KeepAlive) -> {ok, KeepAlive#keepalive{statval = Val}}. + +%% @doc Update keepalive. +%% The statval of the previous keepalive will be used, +%% and normal checks will begin from the next cycle. +-spec update(non_neg_integer(), keepalive() | undefined) -> keepalive() | undefined. +update(Interval, undefined) -> init(0, Interval); +update(Interval, #keepalive{statval = StatVal}) -> init(StatVal, Interval). From d77d5e33bc2143a946131825a27321b9ea3b7bc2 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 18 May 2023 14:48:37 +0800 Subject: [PATCH 126/144] fix(mqttsn): Instantly refresh client info after subscribed/unsubscribed --- apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl index ae1da5dac..914f837e1 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl @@ -1791,14 +1791,14 @@ message_to_packet( handle_call({subscribe, Topic, SubOpts}, _From, Channel) -> case do_subscribe({?SN_INVALID_TOPIC_ID, Topic, SubOpts}, Channel) of {ok, {_, NTopicName, NSubOpts}, NChannel} -> - reply({ok, {NTopicName, NSubOpts}}, NChannel); + reply_and_update({ok, {NTopicName, NSubOpts}}, NChannel); {error, ?SN_RC2_EXCEED_LIMITATION} -> reply({error, exceed_limitation}, Channel) end; handle_call({unsubscribe, Topic}, _From, Channel) -> TopicFilters = [emqx_topic:parse(Topic)], {ok, _, NChannel} = do_unsubscribe(TopicFilters, Channel), - reply(ok, NChannel); + reply_and_update(ok, NChannel); handle_call(subscriptions, _From, Channel = #channel{session = Session}) -> reply({ok, maps:to_list(emqx_session:info(subscriptions, Session))}, Channel); handle_call(kick, _From, Channel) -> @@ -2192,6 +2192,9 @@ terminate(_Reason, _Channel) -> reply(Reply, Channel) -> {reply, Reply, Channel}. +reply_and_update(Reply, Channel) -> + {reply, Reply, [{event, updated}], Channel}. + shutdown(Reason, Channel) -> {shutdown, Reason, Channel}. From c5398ab65192f8133c6228e7ad6d3af6b67a7f9e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 18 May 2023 14:50:22 +0800 Subject: [PATCH 127/144] fix(gateway): avoid uri_decode twice The url path has beed decoded in https://github.com/emqx/cowboy/blob/0ebceb432f46b733fcb54c018ad2aae1e0c80f77/src/cowboy_router.erl#L324-L330. So, we don't need uri_decode in minirest callback again. --- .../src/emqx_gateway_api_clients.erl | 18 ++++++---------- .../src/emqx_gateway_api_listeners.erl | 21 +++++++------------ 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index cd387e3bb..8c29733a0 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -145,10 +145,9 @@ clients(get, #{ clients_insta(get, #{ bindings := #{ name := Name0, - clientid := ClientId0 + clientid := ClientId } }) -> - ClientId = emqx_mgmt_util:urldecode(ClientId0), with_gateway(Name0, fun(GwName, _) -> case emqx_gateway_http:lookup_client( @@ -172,10 +171,9 @@ clients_insta(get, #{ clients_insta(delete, #{ bindings := #{ name := Name0, - clientid := ClientId0 + clientid := ClientId } }) -> - ClientId = emqx_mgmt_util:urldecode(ClientId0), with_gateway(Name0, fun(GwName, _) -> _ = emqx_gateway_http:kickout_client(GwName, ClientId), {204} @@ -185,10 +183,9 @@ clients_insta(delete, #{ subscriptions(get, #{ bindings := #{ name := Name0, - clientid := ClientId0 + clientid := ClientId } }) -> - ClientId = emqx_mgmt_util:urldecode(ClientId0), with_gateway(Name0, fun(GwName, _) -> case emqx_gateway_http:list_client_subscriptions(GwName, ClientId) of {error, not_found} -> @@ -203,11 +200,10 @@ subscriptions(get, #{ subscriptions(post, #{ bindings := #{ name := Name0, - clientid := ClientId0 + clientid := ClientId }, body := Body }) -> - ClientId = emqx_mgmt_util:urldecode(ClientId0), with_gateway(Name0, fun(GwName, _) -> case {maps:get(<<"topic">>, Body, undefined), subopts(Body)} of {undefined, _} -> @@ -231,12 +227,10 @@ subscriptions(post, #{ subscriptions(delete, #{ bindings := #{ name := Name0, - clientid := ClientId0, - topic := Topic0 + clientid := ClientId, + topic := Topic } }) -> - ClientId = emqx_mgmt_util:urldecode(ClientId0), - Topic = emqx_mgmt_util:urldecode(Topic0), with_gateway(Name0, fun(GwName, _) -> _ = emqx_gateway_http:client_unsubscribe(GwName, ClientId, Topic), {204} diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index d90bf3689..2a6d59e35 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -110,14 +110,12 @@ listeners(post, #{bindings := #{name := Name0}, body := LConf}) -> end end). -listeners_insta(delete, #{bindings := #{name := Name0, id := ListenerId0}}) -> - ListenerId = emqx_mgmt_util:urldecode(ListenerId0), +listeners_insta(delete, #{bindings := #{name := Name0, id := ListenerId}}) -> with_gateway(Name0, fun(_GwName, _) -> ok = emqx_gateway_http:remove_listener(ListenerId), {204} end); -listeners_insta(get, #{bindings := #{name := Name0, id := ListenerId0}}) -> - ListenerId = emqx_mgmt_util:urldecode(ListenerId0), +listeners_insta(get, #{bindings := #{name := Name0, id := ListenerId}}) -> with_gateway(Name0, fun(_GwName, _) -> case emqx_gateway_conf:listener(ListenerId) of {ok, Listener} -> @@ -130,9 +128,8 @@ listeners_insta(get, #{bindings := #{name := Name0, id := ListenerId0}}) -> end); listeners_insta(put, #{ body := LConf, - bindings := #{name := Name0, id := ListenerId0} + bindings := #{name := Name0, id := ListenerId} }) -> - ListenerId = emqx_mgmt_util:urldecode(ListenerId0), with_gateway(Name0, fun(_GwName, _) -> {ok, RespConf} = emqx_gateway_http:update_listener(ListenerId, LConf), {200, RespConf} @@ -141,10 +138,9 @@ listeners_insta(put, #{ listeners_insta_authn(get, #{ bindings := #{ name := Name0, - id := ListenerId0 + id := ListenerId } }) -> - ListenerId = emqx_mgmt_util:urldecode(ListenerId0), with_gateway(Name0, fun(GwName, _) -> try emqx_gateway_http:authn(GwName, ListenerId) of Authn -> {200, Authn} @@ -157,10 +153,9 @@ listeners_insta_authn(post, #{ body := Conf, bindings := #{ name := Name0, - id := ListenerId0 + id := ListenerId } }) -> - ListenerId = emqx_mgmt_util:urldecode(ListenerId0), with_gateway(Name0, fun(GwName, _) -> {ok, Authn} = emqx_gateway_http:add_authn(GwName, ListenerId, Conf), {201, Authn} @@ -169,10 +164,9 @@ listeners_insta_authn(put, #{ body := Conf, bindings := #{ name := Name0, - id := ListenerId0 + id := ListenerId } }) -> - ListenerId = emqx_mgmt_util:urldecode(ListenerId0), with_gateway(Name0, fun(GwName, _) -> {ok, Authn} = emqx_gateway_http:update_authn( GwName, ListenerId, Conf @@ -182,10 +176,9 @@ listeners_insta_authn(put, #{ listeners_insta_authn(delete, #{ bindings := #{ name := Name0, - id := ListenerId0 + id := ListenerId } }) -> - ListenerId = emqx_mgmt_util:urldecode(ListenerId0), with_gateway(Name0, fun(GwName, _) -> ok = emqx_gateway_http:remove_authn(GwName, ListenerId), {204} From 26b69a5591d9bff4e07d2dc9cc57db8bf7396b3c Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 18 May 2023 16:16:09 +0800 Subject: [PATCH 128/144] test(mqttsn): add test cases --- .../test/emqx_sn_protocol_SUITE.erl | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl index 04b1b5fb2..cce4ce904 100644 --- a/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl @@ -2259,6 +2259,46 @@ t_clients_subscription_api(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). +t_clients_api_complex_id(_) -> + ClientId = <<"!@#$%^&*()_+{}:\"<>?/">>, + ClientIdUriEncoded = cow_qs:urlencode(ClientId), + Path = "/gateways/mqttsn/clients/" ++ binary_to_list(ClientIdUriEncoded), + {ok, Socket} = gen_udp:open(0, [binary]), + send_connect_msg(Socket, ClientId), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + %% get + {200, Client} = request(get, Path), + ?assertMatch(#{clientid := ClientId}, Client), + %% subscription list + {200, []} = request(get, Path ++ "/subscriptions"), + %% kickout + {204, _} = request(delete, Path), + gen_udp:close(Socket). + +t_update_info_after_subscribed_via_api(_) -> + ClientId = <<"client_id_test1">>, + Path = "/gateways/mqttsn/clients/client_id_test1/subscriptions", + {ok, Socket} = gen_udp:open(0, [binary]), + send_connect_msg(Socket, ClientId), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + %% create + SubReq = #{ + topic => <<"tx">>, + qos => 1, + nl => 0, + rap => 0, + rh => 0 + }, + {201, _SubsResp} = request(post, Path, SubReq), + timer:sleep(500), + %% assert + {200, Client} = request(get, "/gateways/mqttsn/clients/client_id_test1"), + ?assertMatch(#{subscriptions_cnt := 1}, Client), + + send_disconnect_msg(Socket, undefined), + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + gen_udp:close(Socket). + %%-------------------------------------------------------------------- %% Helper funcs %%-------------------------------------------------------------------- From f42cab9c56c5d3e45407137671bc8a245a962247 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 18 May 2023 16:24:37 +0800 Subject: [PATCH 129/144] chore: update changes --- changes/ce/fix-10737.en.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ce/fix-10737.en.md diff --git a/changes/ce/fix-10737.en.md b/changes/ce/fix-10737.en.md new file mode 100644 index 000000000..b5826bac2 --- /dev/null +++ b/changes/ce/fix-10737.en.md @@ -0,0 +1,2 @@ +Fix the issue where the HTTP API interface of Gateway cannot handle ClientIDs with +special characters, such as: `!@#$%^&*()_+{}:"<>?/`. From 52e2caa67164c9a43c41bae86e30dbd89521c8d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Thu, 18 May 2023 21:36:45 +0800 Subject: [PATCH 130/144] fix: check authz's file rule before save to file --- apps/emqx_authz/src/emqx_authz.erl | 34 ++++++++---- apps/emqx_authz/src/emqx_authz_api_schema.erl | 6 +- apps/emqx_authz/src/emqx_authz_file.erl | 2 +- apps/emqx_authz/src/emqx_authz_rule.erl | 8 ++- apps/emqx_authz/test/emqx_authz_SUITE.erl | 55 +++++++++++++++++-- .../emqx_authz/test/emqx_authz_file_SUITE.erl | 4 +- changes/ce/fix-10742.en.md | 2 + 7 files changed, 90 insertions(+), 21 deletions(-) create mode 100644 changes/ce/fix-10742.en.md diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 682ad7f2e..5fffb61d2 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -140,7 +140,12 @@ update(Cmd, Sources) -> emqx_authz_utils:update_config(?CONF_KEY_PATH, {Cmd, Sources}). pre_config_update(_, Cmd, Sources) -> - {ok, do_pre_config_update(Cmd, Sources)}. + try do_pre_config_update(Cmd, Sources) of + {error, Reason} -> {error, Reason}; + NSources -> {ok, NSources} + catch + _:Reason -> {error, Reason} + end. do_pre_config_update({?CMD_MOVE, _, _} = Cmd, Sources) -> do_move(Cmd, Sources); @@ -475,11 +480,12 @@ maybe_write_files(#{<<"type">> := <<"file">>} = Source) -> maybe_write_files(NewSource) -> maybe_write_certs(NewSource). -write_acl_file(#{<<"rules">> := Rules} = Source) -> - NRules = check_acl_file_rules(Rules), - Path = ?MODULE:acl_conf_file(), - {ok, _Filename} = write_file(Path, NRules), - maps:without([<<"rules">>], Source#{<<"path">> => Path}). +write_acl_file(#{<<"rules">> := Rules} = Source0) -> + AclPath = ?MODULE:acl_conf_file(), + ok = check_acl_file_rules(AclPath, Rules), + ok = write_file(AclPath, Rules), + Source1 = maps:remove(<<"rules">>, Source0), + maps:put(<<"path">>, AclPath, Source1). %% @doc where the acl.conf file is stored. acl_conf_file() -> @@ -506,7 +512,7 @@ write_file(Filename, Bytes) -> ok = filelib:ensure_dir(Filename), case file:write_file(Filename, Bytes) of ok -> - {ok, iolist_to_binary(Filename)}; + ok; {error, Reason} -> ?SLOG(error, #{filename => Filename, msg => "write_file_error", reason => Reason}), throw(Reason) @@ -528,6 +534,14 @@ get_source_by_type(Type, Sources) -> update_authz_chain(Actions) -> emqx_hooks:put('client.authorize', {?MODULE, authorize, [Actions]}, ?HP_AUTHZ). -check_acl_file_rules(RawRules) -> - %% TODO: make sure the bin rules checked - RawRules. +check_acl_file_rules(Path, Rules) -> + TmpPath = Path ++ ".tmp", + try + ok = write_file(Path, Rules), + #{annotations := #{rules := _}} = emqx_authz_file:create(#{path => Path}), + ok + catch + throw:Reason -> throw(Reason) + after + _ = file:delete(TmpPath) + end. diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index c89ac5bee..049c84713 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -39,8 +39,10 @@ fields(file) -> type => binary(), required => true, example => - <<"{allow,{username,\"^dashboard?\"},", "subscribe,[\"$SYS/#\"]}.\n", - "{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}.">>, + << + "{allow,{username,{re,\"^dashboard$\"}},subscribe,[\"$SYS/#\"]}.\n", + "{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}." + >>, desc => ?DESC(rules) }} ]; diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl index 54f1775c6..dfd28f0c0 100644 --- a/apps/emqx_authz/src/emqx_authz_file.erl +++ b/apps/emqx_authz/src/emqx_authz_file.erl @@ -54,7 +54,7 @@ create(#{path := Path0} = Source) -> throw(failed_to_read_acl_file); {error, Reason} -> ?SLOG(alert, #{msg => bad_acl_file_content, path => Path, reason => Reason}), - throw(bad_acl_file_content) + throw({bad_acl_file_content, Reason}) end, Source#{annotations => #{rules => Rules}}. diff --git a/apps/emqx_authz/src/emqx_authz_rule.erl b/apps/emqx_authz/src/emqx_authz_rule.erl index bdd0904f7..ec1a8c5de 100644 --- a/apps/emqx_authz/src/emqx_authz_rule.erl +++ b/apps/emqx_authz/src/emqx_authz_rule.erl @@ -68,7 +68,13 @@ compile({Permission, Who, Action, TopicFilters}) when {atom(Permission), compile_who(Who), atom(Action), [ compile_topic(Topic) || Topic <- TopicFilters - ]}. + ]}; +compile({Permission, _Who, _Action, _TopicFilter}) when not ?ALLOW_DENY(Permission) -> + throw({invalid_authorization_permission, Permission}); +compile({_Permission, _Who, Action, _TopicFilter}) when not ?PUBSUB(Action) -> + throw({invalid_authorization_action, Action}); +compile(BadRule) -> + throw({invalid_authorization_rule, BadRule}). compile_who(all) -> all; diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index 84b1d903e..2c57d807a 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -155,22 +155,36 @@ set_special_configs(_App) -> <<"ssl">> => #{<<"enable">> => false}, <<"cmd">> => <<"HGETALL mqtt_authz:", ?PH_USERNAME/binary>> }). --define(SOURCE6, #{ + +-define(FILE_SOURCE(Rules), #{ <<"type">> => <<"file">>, <<"enable">> => true, - <<"rules">> => + <<"rules">> => Rules +}). + +-define(SOURCE6, + ?FILE_SOURCE( << "{allow,{username,\"^dashboard?\"},subscribe,[\"$SYS/#\"]}." "\n{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}." >> -}). --define(SOURCE7, #{ + ) +). +-define(SOURCE7, + ?FILE_SOURCE( + << + "{allow,{username,\"some_client\"},publish,[\"some_client/lwt\"]}.\n" + "{deny, all}." + >> + ) +). + +-define(BAD_FILE_SOURCE2, #{ <<"type">> => <<"file">>, <<"enable">> => true, <<"rules">> => << - "{allow,{username,\"some_client\"},publish,[\"some_client/lwt\"]}.\n" - "{deny, all}." + "{not_allow,{username,\"some_client\"},publish,[\"some_client/lwt\"]}." >> }). @@ -178,6 +192,35 @@ set_special_configs(_App) -> %% Testcases %%------------------------------------------------------------------------------ +-define(UPDATE_ERROR(Err), {error, {pre_config_update, emqx_authz, Err}}). + +t_bad_file_source(_) -> + BadContent = ?FILE_SOURCE(<<"{allow,{username,\"bar\"}, publish, [\"test\"]}">>), + BadContentErr = {bad_acl_file_content, {1, erl_parse, ["syntax error before: ", []]}}, + BadRule = ?FILE_SOURCE(<<"{allow,{username,\"bar\"},publish}.">>), + BadRuleErr = {invalid_authorization_rule, {allow, {username, "bar"}, publish}}, + BadPermission = ?FILE_SOURCE(<<"{not_allow,{username,\"bar\"},publish,[\"test\"]}.">>), + BadPermissionErr = {invalid_authorization_permission, not_allow}, + BadAction = ?FILE_SOURCE(<<"{allow,{username,\"bar\"},pubsub,[\"test\"]}.">>), + BadActionErr = {invalid_authorization_action, pubsub}, + lists:foreach( + fun({Source, Error}) -> + ?assertEqual(?UPDATE_ERROR(Error), emqx_authz:update(?CMD_REPLACE, [Source])), + ?assertEqual(?UPDATE_ERROR(Error), emqx_authz:update(?CMD_PREPEND, Source)), + ?assertEqual(?UPDATE_ERROR(Error), emqx_authz:update(?CMD_APPEND, Source)) + end, + [ + {BadContent, BadContentErr}, + {BadRule, BadRuleErr}, + {BadPermission, BadPermissionErr}, + {BadAction, BadActionErr} + ] + ), + ?assertMatch( + [], + emqx_conf:get([authorization, sources], []) + ). + t_update_source(_) -> %% replace all {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE3]), diff --git a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl index 124fe904f..be8907feb 100644 --- a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl @@ -120,7 +120,9 @@ t_superuser(_Config) -> t_invalid_file(_Config) -> ?assertMatch( - {error, bad_acl_file_content}, + {error, + {pre_config_update, emqx_authz, + {bad_acl_file_content, {1, erl_parse, ["syntax error before: ", "term"]}}}}, emqx_authz:update(?CMD_REPLACE, [?RAW_SOURCE#{<<"rules">> => <<"{{invalid term">>}]) ). diff --git a/changes/ce/fix-10742.en.md b/changes/ce/fix-10742.en.md new file mode 100644 index 000000000..cc8232a04 --- /dev/null +++ b/changes/ce/fix-10742.en.md @@ -0,0 +1,2 @@ +Check the correctness of the rules before saving the authorization file source. +Previously, Saving wrong rules could lead to restart failure. From 5c112602b8e48439f1e229c924f9667930524043 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 19 May 2023 13:37:36 +0300 Subject: [PATCH 131/144] test(evict): fix flapping testcase Postpone enabling eviction agent so that target connection will not get evicted too early. --- apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl index 22b694d77..a68a1f292 100644 --- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl +++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl @@ -159,8 +159,6 @@ t_explicit_session_takeover(Config) -> ]), {ok, _, _} = emqtt:subscribe(C0, <<"t1">>), - ok = rpc:call(Node1, emqx_eviction_agent, enable, [test_eviction, undefined]), - ?assertEqual( 1, rpc:call(Node1, emqx_eviction_agent, connection_count, []) @@ -168,6 +166,8 @@ t_explicit_session_takeover(Config) -> [ChanPid] = rpc:call(Node1, emqx_cm, lookup_channels, [<<"client_with_session">>]), + ok = rpc:call(Node1, emqx_eviction_agent, enable, [test_eviction, undefined]), + ?assertWaitEvent( begin ok = rpc:call(Node1, emqx_eviction_agent, evict_connections, [1]), From 6cb9efd7d3630900e8a8733db9e17f0d1936e32f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Fri, 19 May 2023 08:55:04 +0800 Subject: [PATCH 132/144] feat: add authz file rule validator --- apps/emqx/test/emqx_common_test_helpers.erl | 12 ++++++++++ apps/emqx_authz/src/emqx_authz.erl | 5 ++-- apps/emqx_authz/src/emqx_authz_schema.erl | 24 ++++++++++++++++--- .../emqx_conf/test/emqx_conf_schema_tests.erl | 14 +++++++++++ 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index c8ef40925..d8e0690f2 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -237,6 +237,8 @@ render_and_load_app_config(App, Opts) -> end. do_render_app_config(App, Schema, ConfigFile, Opts) -> + %% copy acl_conf must run before read_schema_configs + copy_acl_conf(), Vars = mustache_vars(App, Opts), RenderedConfigFile = render_config_file(ConfigFile, Vars), read_schema_configs(Schema, RenderedConfigFile), @@ -520,6 +522,16 @@ copy_certs(emqx_conf, Dest0) -> copy_certs(_, _) -> ok. +copy_acl_conf() -> + Dest = filename:join([code:lib_dir(emqx), "etc/acl.conf"]), + case code:lib_dir(emqx_authz) of + {error, bad_name} -> + (not filelib:is_regular(Dest)) andalso file:write_file(Dest, <<"">>); + _ -> + {ok, _} = file:copy(deps_path(emqx_authz, "etc/acl.conf"), Dest) + end, + ok. + load_config(SchemaModule, Config) -> ConfigBin = case is_map(Config) of diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 5fffb61d2..e7f59cb71 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -482,6 +482,8 @@ maybe_write_files(NewSource) -> write_acl_file(#{<<"rules">> := Rules} = Source0) -> AclPath = ?MODULE:acl_conf_file(), + %% Always check if the rules are valid before writing to the file + %% If the rules are invalid, the old file will be kept ok = check_acl_file_rules(AclPath, Rules), ok = write_file(AclPath, Rules), Source1 = maps:remove(<<"rules">>, Source0), @@ -538,8 +540,7 @@ check_acl_file_rules(Path, Rules) -> TmpPath = Path ++ ".tmp", try ok = write_file(Path, Rules), - #{annotations := #{rules := _}} = emqx_authz_file:create(#{path => Path}), - ok + emqx_authz_schema:validate_file_rules(Path) catch throw:Reason -> throw(Reason) after diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 26a22f73b..5f06f757c 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -42,7 +42,8 @@ -export([ headers_no_content_type/1, - headers/1 + headers/1, + validate_file_rules/1 ]). %%-------------------------------------------------------------------- @@ -78,7 +79,17 @@ fields("authorization") -> authz_fields(); fields(file) -> authz_common_fields(file) ++ - [{path, ?HOCON(string(), #{required => true, desc => ?DESC(path)})}]; + [ + {path, + ?HOCON( + string(), + #{ + required => true, + validator => fun ?MODULE:validate_file_rules/1, + desc => ?DESC(path) + } + )} + ]; fields(http_get) -> authz_common_fields(http) ++ http_common_fields() ++ @@ -495,7 +506,7 @@ authz_fields() -> %% doc_lift is force a root level reference instead of nesting sub-structs extra => #{doc_lift => true}, %% it is recommended to configure authz sources from dashboard - %% hance the importance level for config is low + %% hence the importance level for config is low importance => ?IMPORTANCE_LOW } )} @@ -507,3 +518,10 @@ default_authz() -> <<"enable">> => true, <<"path">> => <<"${EMQX_ETC_DIR}/acl.conf">> }. + +validate_file_rules(Path) -> + %% Don't need assert the create result here, all error is thrown + %% some test mock the create function + %% #{annotations := #{rules := _}} + _ = emqx_authz_file:create(#{path => Path}), + ok. diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 5aa45d9ad..b59a5f819 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -23,6 +23,7 @@ """). array_nodes_test() -> + ensure_acl_conf(), ExpectNodes = ['emqx1@127.0.0.1', 'emqx2@127.0.0.1'], lists:foreach( fun(Nodes) -> @@ -133,6 +134,7 @@ outdated_log_test() -> validate_log(?OUTDATED_LOG_CONF). validate_log(Conf) -> + ensure_acl_conf(), BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), Conf0 = <>, {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), @@ -199,6 +201,7 @@ log_test() -> %% erlfmt-ignore log_rotation_count_limit_test() -> + ensure_acl_conf(), Format = """ log.file { @@ -271,6 +274,7 @@ log_rotation_count_limit_test() -> ). authn_validations_test() -> + ensure_acl_conf(), BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), OKHttps = to_bin(?BASE_AUTHN_ARRAY, [post, true, <<"https://127.0.0.1:8080">>]), @@ -328,6 +332,7 @@ authn_validations_test() -> ). listeners_test() -> + ensure_acl_conf(), BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), Conf = <>, @@ -402,6 +407,7 @@ authentication_headers(Conf) -> Headers. doc_gen_test() -> + ensure_acl_conf(), %% the json file too large to encode. { timeout, @@ -424,3 +430,11 @@ doc_gen_test() -> to_bin(Format, Args) -> iolist_to_binary(io_lib:format(Format, Args)). + +ensure_acl_conf() -> + File = emqx_schema:naive_env_interpolation(<<"${EMQX_ETC_DIR}/acl.conf">>), + ok = filelib:ensure_dir(filename:dirname(File)), + case filelib:is_regular(File) of + true -> ok; + false -> file:write_file(File, <<"">>) + end. From c440cd77b017912a9c10cbac5b88f8928ca466b3 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 19 May 2023 15:37:01 +0300 Subject: [PATCH 133/144] fix(test): prefer `emqx_connector_http:join_paths/2` in api tests It is more consistent with what HTTP servers are expecting, plus not OS-specific as `filename:join/2`. --- apps/emqx_management/test/emqx_mgmt_api_test_util.erl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index 985b95d5b..c62d27904 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -146,10 +146,15 @@ build_http_header(X) -> [X]. api_path(Parts) -> - ?SERVER ++ filename:join([?BASE_PATH | Parts]). + join_http_path([?SERVER, ?BASE_PATH | Parts]). api_path_without_base_path(Parts) -> - ?SERVER ++ filename:join([Parts]). + join_http_path([?SERVER | Parts]). + +join_http_path([]) -> + []; +join_http_path([Part | Rest]) -> + lists:foldl(fun(P, Acc) -> emqx_connector_http:join_paths(Acc, P) end, Part, Rest). %% Usage: %% upload_request(<<"site.com/api/upload">>, <<"path/to/file.png">>, From ba6b208df2a0f2fbcbc489e0607903f9569936b6 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 19 May 2023 23:08:40 +0300 Subject: [PATCH 134/144] fix(clickhouse): start app in tests Otherwise, depending on the test execution order, tests might sometimes fail. Moreover, ensure that applications describe their dependecies correctly and avoid starting irrelevant apps in tests. --- .../src/emqx_bridge_clickhouse.app.src | 2 +- .../src/emqx_bridge_clickhouse.erl | 1 - .../test/emqx_bridge_clickhouse_SUITE.erl | 13 ++++--------- .../test/emqx_bridge_clickhouse_connector_SUITE.erl | 9 +++------ apps/emqx_resource/src/emqx_resource.app.src | 1 + 5 files changed, 9 insertions(+), 17 deletions(-) diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src index 751628fa4..72669ba8f 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src @@ -2,7 +2,7 @@ {description, "EMQX Enterprise ClickHouse Bridge"}, {vsn, "0.2.0"}, {registered, []}, - {applications, [kernel, stdlib, clickhouse]}, + {applications, [kernel, stdlib, clickhouse, emqx_resource]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl index d317ecc5c..9abcadbba 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl @@ -3,7 +3,6 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_clickhouse). --include_lib("emqx_bridge/include/emqx_bridge.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). diff --git a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl index b39296044..787fb81ff 100644 --- a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl @@ -7,6 +7,7 @@ -compile(nowarn_export_all). -compile(export_all). +-define(APP, emqx_bridge_clickhouse). -define(CLICKHOUSE_HOST, "clickhouse"). -include_lib("emqx_connector/include/emqx_connector.hrl"). @@ -25,10 +26,7 @@ init_per_suite(Config) -> true -> emqx_common_test_helpers:render_and_load_app_config(emqx_conf), ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), - ok = emqx_connector_test_helpers:start_apps([emqx_resource]), - {ok, _} = application:ensure_all_started(emqx_connector), - {ok, _} = application:ensure_all_started(emqx_ee_connector), - {ok, _} = application:ensure_all_started(emqx_ee_bridge), + ok = emqx_connector_test_helpers:start_apps([emqx_resource, ?APP]), snabbkaffe:fix_ct_logging(), %% Create the db table Conn = start_clickhouse_connection(), @@ -75,11 +73,8 @@ start_clickhouse_connection() -> end_per_suite(Config) -> ClickhouseConnection = proplists:get_value(clickhouse_connection, Config), clickhouse:stop(ClickhouseConnection), - ok = emqx_common_test_helpers:stop_apps([emqx_conf]), - ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), - _ = application:stop(emqx_connector), - _ = application:stop(emqx_ee_connector), - _ = application:stop(emqx_bridge). + ok = emqx_connector_test_helpers:stop_apps([?APP, emqx_resource]), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]). init_per_testcase(_, Config) -> reset_table(Config), diff --git a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl index 549a0d870..0bae413e0 100644 --- a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl @@ -9,9 +9,9 @@ -include("emqx_connector.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/assert.hrl"). +-define(APP, emqx_bridge_clickhouse). -define(CLICKHOUSE_HOST, "clickhouse"). -define(CLICKHOUSE_RESOURCE_MOD, emqx_bridge_clickhouse_connector). @@ -51,9 +51,7 @@ init_per_suite(Config) -> of true -> ok = emqx_common_test_helpers:start_apps([emqx_conf]), - ok = emqx_connector_test_helpers:start_apps([emqx_resource]), - {ok, _} = application:ensure_all_started(emqx_connector), - {ok, _} = application:ensure_all_started(emqx_ee_connector), + ok = emqx_connector_test_helpers:start_apps([emqx_resource, ?APP]), %% Create the db table {ok, Conn} = clickhouse:start_link([ @@ -76,8 +74,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_conf]), - ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), - _ = application:stop(emqx_connector). + ok = emqx_connector_test_helpers:stop_apps([?APP, emqx_resource]). init_per_testcase(_, Config) -> Config. diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 3b92f1200..7c1b5ad2a 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -9,6 +9,7 @@ stdlib, gproc, jsx, + ecpool, emqx, telemetry ]}, From 96e7005de87ca0f42dfdc1aab4cf8cf021cf57c6 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 22 May 2023 11:03:23 +0800 Subject: [PATCH 135/144] refactor: add emqx_authz_file validate function --- apps/emqx_authz/src/emqx_authz.erl | 5 +++-- apps/emqx_authz/src/emqx_authz_file.erl | 7 ++++++- apps/emqx_authz/src/emqx_authz_schema.erl | 12 ++---------- apps/emqx_authz/test/emqx_authz_SUITE.erl | 7 ++++++- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index e7f59cb71..c7db65992 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -539,8 +539,9 @@ update_authz_chain(Actions) -> check_acl_file_rules(Path, Rules) -> TmpPath = Path ++ ".tmp", try - ok = write_file(Path, Rules), - emqx_authz_schema:validate_file_rules(Path) + ok = write_file(TmpPath, Rules), + {ok, _} = emqx_authz_file:validate(TmpPath), + ok catch throw:Reason -> throw(Reason) after diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl index dfd28f0c0..317395a45 100644 --- a/apps/emqx_authz/src/emqx_authz_file.erl +++ b/apps/emqx_authz/src/emqx_authz_file.erl @@ -33,13 +33,14 @@ update/1, destroy/1, authorize/4, + validate/1, read_file/1 ]). description() -> "AuthZ with static rules". -create(#{path := Path0} = Source) -> +validate(Path0) -> Path = filename(Path0), Rules = case file:consult(Path) of @@ -56,6 +57,10 @@ create(#{path := Path0} = Source) -> ?SLOG(alert, #{msg => bad_acl_file_content, path => Path, reason => Reason}), throw({bad_acl_file_content, Reason}) end, + {ok, Rules}. + +create(#{path := Path} = Source) -> + {ok, Rules} = validate(Path), Source#{annotations => #{rules => Rules}}. update(#{path := _Path} = Source) -> diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 5f06f757c..8e847b93e 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -42,8 +42,7 @@ -export([ headers_no_content_type/1, - headers/1, - validate_file_rules/1 + headers/1 ]). %%-------------------------------------------------------------------- @@ -85,7 +84,7 @@ fields(file) -> string(), #{ required => true, - validator => fun ?MODULE:validate_file_rules/1, + validator => fun(Path) -> element(1, emqx_authz_file:validate(Path)) end, desc => ?DESC(path) } )} @@ -518,10 +517,3 @@ default_authz() -> <<"enable">> => true, <<"path">> => <<"${EMQX_ETC_DIR}/acl.conf">> }. - -validate_file_rules(Path) -> - %% Don't need assert the create result here, all error is thrown - %% some test mock the create function - %% #{annotations := #{rules := _}} - _ = emqx_authz_file:create(#{path => Path}), - ok. diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index 2c57d807a..9c1b7fd51 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -205,9 +205,14 @@ t_bad_file_source(_) -> BadActionErr = {invalid_authorization_action, pubsub}, lists:foreach( fun({Source, Error}) -> + File = emqx_authz:acl_conf_file(), + {ok, Bin1} = file:read_file(File), ?assertEqual(?UPDATE_ERROR(Error), emqx_authz:update(?CMD_REPLACE, [Source])), ?assertEqual(?UPDATE_ERROR(Error), emqx_authz:update(?CMD_PREPEND, Source)), - ?assertEqual(?UPDATE_ERROR(Error), emqx_authz:update(?CMD_APPEND, Source)) + ?assertEqual(?UPDATE_ERROR(Error), emqx_authz:update(?CMD_APPEND, Source)), + %% Check file content not changed if update failed + {ok, Bin2} = file:read_file(File), + ?assertEqual(Bin1, Bin2) end, [ {BadContent, BadContentErr}, From 23542d1262916d43d623b9c6eb8d18ef25ef470e Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 19 May 2023 15:39:22 +0300 Subject: [PATCH 136/144] fix(listen): wait until port is free when stopping ranch listeners Simple `cowboy:stop_listener/1` will not close the listening socket explicitly, it was the source of test flaps before this fix. --- apps/emqx/src/emqx_listeners.erl | 58 +++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 2b80000dc..acdb9ff96 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -277,9 +277,8 @@ restart_listener(Type, ListenerName, Conf) -> restart_listener(Type, ListenerName, Conf, Conf). restart_listener(Type, ListenerName, OldConf, NewConf) -> - case do_stop_listener(Type, ListenerName, OldConf) of + case stop_listener(Type, ListenerName, OldConf) of ok -> start_listener(Type, ListenerName, NewConf); - {error, not_found} -> start_listener(Type, ListenerName, NewConf); {error, Reason} -> {error, Reason} end. @@ -296,42 +295,63 @@ stop_listener(ListenerId) -> apply_on_listener(ListenerId, fun stop_listener/3). stop_listener(Type, ListenerName, #{bind := Bind} = Conf) -> - case do_stop_listener(Type, ListenerName, Conf) of + Id = listener_id(Type, ListenerName), + ok = del_limiter_bucket(Id, Conf), + case do_stop_listener(Type, Id, Conf) of ok -> console_print( "Listener ~ts on ~ts stopped.~n", - [listener_id(Type, ListenerName), format_bind(Bind)] + [Id, format_bind(Bind)] ), ok; {error, not_found} -> - ?ELOG( - "Failed to stop listener ~ts on ~ts: ~0p~n", - [listener_id(Type, ListenerName), format_bind(Bind), already_stopped] - ), ok; {error, Reason} -> ?ELOG( "Failed to stop listener ~ts on ~ts: ~0p~n", - [listener_id(Type, ListenerName), format_bind(Bind), Reason] + [Id, format_bind(Bind), Reason] ), {error, Reason} end. -spec do_stop_listener(atom(), atom(), map()) -> ok | {error, term()}. -do_stop_listener(Type, ListenerName, #{bind := ListenOn} = Conf) when Type == tcp; Type == ssl -> - Id = listener_id(Type, ListenerName), - del_limiter_bucket(Id, Conf), +do_stop_listener(Type, Id, #{bind := ListenOn}) when Type == tcp; Type == ssl -> esockd:close(Id, ListenOn); -do_stop_listener(Type, ListenerName, Conf) when Type == ws; Type == wss -> - Id = listener_id(Type, ListenerName), - del_limiter_bucket(Id, Conf), - cowboy:stop_listener(Id); -do_stop_listener(quic, ListenerName, Conf) -> - Id = listener_id(quic, ListenerName), - del_limiter_bucket(Id, Conf), +do_stop_listener(Type, Id, #{bind := ListenOn}) when Type == ws; Type == wss -> + case cowboy:stop_listener(Id) of + ok -> + wait_listener_stopped(ListenOn); + Error -> + Error + end; +do_stop_listener(quic, Id, _Conf) -> quicer:stop_listener(Id). +wait_listener_stopped(ListenOn) -> + % NOTE + % `cowboy:stop_listener/1` will not close the listening socket explicitly, + % it will be closed by the runtime system **only after** the process exits. + Endpoint = maps:from_list(ip_port(ListenOn)), + case + gen_tcp:connect( + maps:get(ip, Endpoint, loopback), + maps:get(port, Endpoint), + [{active, false}] + ) + of + {error, _EConnrefused} -> + %% NOTE + %% We should get `econnrefused` here because acceptors are already dead + %% but don't want to crash if not, because this doesn't make any difference. + ok; + {ok, Socket} -> + %% NOTE + %% Tiny chance to get a connected socket here, when some other process + %% concurrently binds to the same port. + gen_tcp:close(Socket) + end. + -ifndef(TEST). console_print(Fmt, Args) -> ?ULOG(Fmt, Args). -else. From 7d798c10e9e5dbb12687fb3cc988944e6c4f832e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 17 May 2023 15:23:42 -0300 Subject: [PATCH 137/144] perf(buffer_worker): flush metrics periodically inside buffer worker process Fixes https://emqx.atlassian.net/browse/EMQX-9905 Since calling `telemetry` is costly in a hot path, we instead collect metrics inside the buffer workers state and periodically flush them, rather than immediately as events happen. --- .../test/emqx_bridge_mqtt_SUITE.erl | 26 +- .../test/emqx_bridge_gcp_pubsub_SUITE.erl | 57 +-- apps/emqx_resource/include/emqx_resource.hrl | 4 + .../src/emqx_resource_buffer_worker.erl | 383 +++++++++++------- .../src/schema/emqx_resource_schema.erl | 6 + .../test/emqx_resource_SUITE.erl | 86 +++- 6 files changed, 341 insertions(+), 221 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index bd5cda3f0..f0de07da2 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -100,17 +100,21 @@ ?assertMetrics(Pat, true, BridgeID) ). -define(assertMetrics(Pat, Guard, BridgeID), - ?assertMatch( - #{ - <<"metrics">> := Pat, - <<"node_metrics">> := [ - #{ - <<"node">> := _, - <<"metrics">> := Pat - } - ] - } when Guard, - request_bridge_metrics(BridgeID) + ?retry( + _Sleep = 300, + _Attempts0 = 20, + ?assertMatch( + #{ + <<"metrics">> := Pat, + <<"node_metrics">> := [ + #{ + <<"node">> := _, + <<"metrics">> := Pat + } + ] + } when Guard, + request_bridge_metrics(BridgeID) + ) ) ). diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl index 55527bf1f..49ca57c42 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl @@ -288,6 +288,7 @@ gcp_pubsub_config(Config) -> " pipelining = ~b\n" " resource_opts = {\n" " request_timeout = 500ms\n" + " metrics_flush_interval = 700ms\n" " worker_pool_size = 1\n" " query_mode = ~s\n" " batch_size = ~b\n" @@ -529,12 +530,14 @@ wait_until_gauge_is(GaugeName, ExpectedValue, Timeout) -> end. receive_all_events(EventName, Timeout) -> - receive_all_events(EventName, Timeout, []). + receive_all_events(EventName, Timeout, _MaxEvents = 10, _Count = 0, _Acc = []). -receive_all_events(EventName, Timeout, Acc) -> +receive_all_events(_EventName, _Timeout, MaxEvents, Count, Acc) when Count >= MaxEvents -> + lists:reverse(Acc); +receive_all_events(EventName, Timeout, MaxEvents, Count, Acc) -> receive {telemetry, #{name := [_, _, EventName]} = Event} -> - receive_all_events(EventName, Timeout, [Event | Acc]) + receive_all_events(EventName, Timeout, MaxEvents, Count + 1, [Event | Acc]) after Timeout -> lists:reverse(Acc) end. @@ -557,8 +560,9 @@ wait_n_events(_TelemetryTable, _ResourceId, NEvents, _Timeout, _EventName) when ok; wait_n_events(TelemetryTable, ResourceId, NEvents, Timeout, EventName) -> receive - {telemetry, #{name := [_, _, EventName]}} -> - wait_n_events(TelemetryTable, ResourceId, NEvents - 1, Timeout, EventName) + {telemetry, #{name := [_, _, EventName], measurements := #{counter_inc := Inc}} = Event} -> + ct:pal("telemetry event: ~p", [Event]), + wait_n_events(TelemetryTable, ResourceId, NEvents - Inc, Timeout, EventName) after Timeout -> RecordedEvents = ets:tab2list(TelemetryTable), CurrentMetrics = current_metrics(ResourceId), @@ -575,7 +579,6 @@ t_publish_success(Config) -> ResourceId = ?config(resource_id, Config), ServiceAccountJSON = ?config(service_account_json, Config), TelemetryTable = ?config(telemetry_table, Config), - QueryMode = ?config(query_mode, Config), Topic = <<"t/topic">>, ?check_trace( create_bridge(Config), @@ -604,17 +607,6 @@ t_publish_success(Config) -> ), %% to avoid test flakiness wait_telemetry_event(TelemetryTable, success, ResourceId), - ExpectedInflightEvents = - case QueryMode of - sync -> 1; - async -> 3 - end, - wait_telemetry_event( - TelemetryTable, - inflight, - ResourceId, - #{n_events => ExpectedInflightEvents, timeout => 5_000} - ), wait_until_gauge_is(queuing, 0, 500), wait_until_gauge_is(inflight, 0, 500), assert_metrics( @@ -635,7 +627,6 @@ t_publish_success_local_topic(Config) -> ResourceId = ?config(resource_id, Config), ServiceAccountJSON = ?config(service_account_json, Config), TelemetryTable = ?config(telemetry_table, Config), - QueryMode = ?config(query_mode, Config), LocalTopic = <<"local/topic">>, {ok, _} = create_bridge(Config, #{<<"local_topic">> => LocalTopic}), assert_empty_metrics(ResourceId), @@ -654,17 +645,6 @@ t_publish_success_local_topic(Config) -> ), %% to avoid test flakiness wait_telemetry_event(TelemetryTable, success, ResourceId), - ExpectedInflightEvents = - case QueryMode of - sync -> 1; - async -> 3 - end, - wait_telemetry_event( - TelemetryTable, - inflight, - ResourceId, - #{n_events => ExpectedInflightEvents, timeout => 5_000} - ), wait_until_gauge_is(queuing, 0, 500), wait_until_gauge_is(inflight, 0, 500), assert_metrics( @@ -696,7 +676,6 @@ t_publish_templated(Config) -> ResourceId = ?config(resource_id, Config), ServiceAccountJSON = ?config(service_account_json, Config), TelemetryTable = ?config(telemetry_table, Config), - QueryMode = ?config(query_mode, Config), Topic = <<"t/topic">>, PayloadTemplate = << "{\"payload\": \"${payload}\"," @@ -742,17 +721,6 @@ t_publish_templated(Config) -> ), %% to avoid test flakiness wait_telemetry_event(TelemetryTable, success, ResourceId), - ExpectedInflightEvents = - case QueryMode of - sync -> 1; - async -> 3 - end, - wait_telemetry_event( - TelemetryTable, - inflight, - ResourceId, - #{n_events => ExpectedInflightEvents, timeout => 5_000} - ), wait_until_gauge_is(queuing, 0, 500), wait_until_gauge_is(inflight, 0, 500), assert_metrics( @@ -1089,9 +1057,6 @@ do_econnrefused_or_timeout_test(Config, Error) -> %% message as dropped; and since it never considers the %% response expired, this succeeds. econnrefused -> - wait_telemetry_event(TelemetryTable, queuing, ResourceId, #{ - timeout => 10_000, n_events => 1 - }), %% even waiting, hard to avoid flakiness... simpler to just sleep %% a bit until stabilization. ct:sleep(200), @@ -1111,8 +1076,8 @@ do_econnrefused_or_timeout_test(Config, Error) -> CurrentMetrics ); timeout -> - wait_until_gauge_is(inflight, 0, _Timeout = 400), - wait_until_gauge_is(queuing, 0, _Timeout = 400), + wait_until_gauge_is(inflight, 0, _Timeout = 1_000), + wait_until_gauge_is(queuing, 0, _Timeout = 1_000), assert_metrics( #{ dropped => 0, diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index e6f86fb59..7f3ac580d 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -103,6 +103,10 @@ -define(HEALTHCHECK_INTERVAL, 15000). -define(HEALTHCHECK_INTERVAL_RAW, <<"15s">>). +%% milliseconds +-define(DEFAULT_METRICS_FLUSH_INTERVAL, 5_000). +-define(DEFAULT_METRICS_FLUSH_INTERVAL_RAW, <<"5s">>). + %% milliseconds -define(START_TIMEOUT, 5000). -define(START_TIMEOUT_RAW, <<"5s">>). diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 6145c3d87..993e69749 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -80,16 +80,30 @@ -type health_check_interval() :: timer:time(). -type state() :: blocked | running. -type inflight_key() :: integer(). +-type counters() :: #{ + dropped_expired => non_neg_integer(), + dropped_queue_full => non_neg_integer(), + dropped_resource_not_found => non_neg_integer(), + dropped_resource_stopped => non_neg_integer(), + success => non_neg_integer(), + failed => non_neg_integer(), + retried_success => non_neg_integer(), + retried_failed => non_neg_integer() +}. +-type inflight_table() :: ets:tid() | atom() | reference(). -type data() :: #{ id := id(), index := index(), - inflight_tid := ets:tid(), + inflight_tid := inflight_table(), async_workers := #{pid() => reference()}, batch_size := pos_integer(), batch_time := timer:time(), + counters := counters(), + metrics_flush_interval := timer:time(), queue := replayq:q(), resume_interval := timer:time(), - tref := undefined | timer:tref() + tref := undefined | {timer:tref() | reference(), reference()}, + metrics_tref := undefined | {timer:tref() | reference(), reference()} }. callback_mode() -> [state_functions, state_enter]. @@ -171,24 +185,29 @@ init({Id, Index, Opts}) -> emqx_resource_metrics:queuing_set(Id, Index, queue_count(Queue)), emqx_resource_metrics:inflight_set(Id, Index, 0), InflightWinSize = maps:get(inflight_window, Opts, ?DEFAULT_INFLIGHT), - InflightTID = inflight_new(InflightWinSize, Id, Index), + InflightTID = inflight_new(InflightWinSize), HealthCheckInterval = maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL), RequestTimeout = maps:get(request_timeout, Opts, ?DEFAULT_REQUEST_TIMEOUT), BatchTime0 = maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME), BatchTime = adjust_batch_time(Id, RequestTimeout, BatchTime0), DefaultResumeInterval = default_resume_interval(RequestTimeout, HealthCheckInterval), ResumeInterval = maps:get(resume_interval, Opts, DefaultResumeInterval), - Data = #{ + MetricsFlushInterval = maps:get(metrics_flush_interval, Opts, ?DEFAULT_METRICS_FLUSH_INTERVAL), + Data0 = #{ id => Id, index => Index, inflight_tid => InflightTID, async_workers => #{}, batch_size => BatchSize, batch_time => BatchTime, + counters => #{}, + metrics_flush_interval => MetricsFlushInterval, queue => Queue, resume_interval => ResumeInterval, - tref => undefined + tref => undefined, + metrics_tref => undefined }, + Data = ensure_metrics_flush_timer(Data0), ?tp(buffer_worker_init, #{id => Id, index => Index, queue_opts => QueueOpts}), {ok, running, Data}. @@ -208,11 +227,16 @@ running(cast, block, St) -> {next_state, blocked, St}; running(info, ?SEND_REQ(_ReplyTo, _Req) = Request0, Data) -> handle_query_requests(Request0, Data); -running(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) -> - flush(St#{tref := undefined}); -running(info, {flush, _Ref}, _St) -> +running(info, {flush, Ref}, Data = #{tref := {_TRef, Ref}}) -> + flush(Data#{tref := undefined}); +running(info, {flush, _Ref}, _Data) -> ?tp(discarded_stale_flush, #{}), keep_state_and_data; +running(info, {flush_metrics, Ref}, Data0 = #{metrics_tref := {_TRef, Ref}}) -> + Data = flush_metrics(Data0#{metrics_tref := undefined}), + {keep_state, Data}; +running(info, {flush_metrics, _Ref}, _Data) -> + keep_state_and_data; running(info, {'DOWN', _MRef, process, Pid, Reason}, Data0 = #{async_workers := AsyncWorkers0}) when is_map_key(Pid, AsyncWorkers0) -> @@ -241,6 +265,11 @@ blocked(info, ?SEND_REQ(_ReplyTo, _Req) = Request0, Data0) -> blocked(info, {flush, _Ref}, _Data) -> %% ignore stale timer keep_state_and_data; +blocked(info, {flush_metrics, Ref}, Data0 = #{metrics_tref := {_TRef, Ref}}) -> + Data = flush_metrics(Data0#{metrics_tref := undefined}), + {keep_state, Data}; +blocked(info, {flush_metrics, _Ref}, _Data) -> + keep_state_and_data; blocked(info, {'DOWN', _MRef, process, Pid, Reason}, Data0 = #{async_workers := AsyncWorkers0}) when is_map_key(Pid, AsyncWorkers0) -> @@ -310,11 +339,7 @@ pick_cast(Id, Key, Query) -> resume_from_blocked(Data) -> ?tp(buffer_worker_resume_from_blocked_enter, #{}), - #{ - id := Id, - index := Index, - inflight_tid := InflightTID - } = Data, + #{inflight_tid := InflightTID} = Data, Now = now_(), case inflight_get_first_retriable(InflightTID, Now) of none -> @@ -326,10 +351,15 @@ resume_from_blocked(Data) -> end; {expired, Ref, Batch} -> WorkerPid = self(), - IsAcked = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), - IsAcked andalso emqx_resource_metrics:dropped_expired_inc(Id, length(Batch)), + IsAcked = ack_inflight(InflightTID, Ref, WorkerPid), + Counters = + case IsAcked of + true -> #{dropped_expired => length(Batch)}; + false -> #{} + end, + NData = aggregate_counters(Data, Counters), ?tp(buffer_worker_retry_expired, #{expired => Batch}), - resume_from_blocked(Data); + resume_from_blocked(NData); {single, Ref, Query} -> %% We retry msgs in inflight window sync, as if we send them %% async, they will be appended to the end of inflight window again. @@ -339,11 +369,11 @@ resume_from_blocked(Data) -> {batch, Ref, NotExpired, Expired} -> NumExpired = length(Expired), ok = update_inflight_item(InflightTID, Ref, NotExpired, NumExpired), - emqx_resource_metrics:dropped_expired_inc(Id, NumExpired), + NData = aggregate_counters(Data, #{dropped_expired => NumExpired}), ?tp(buffer_worker_retry_expired, #{expired => Expired}), %% We retry msgs in inflight window sync, as if we send them %% async, they will be appended to the end of inflight window again. - retry_inflight_sync(Ref, NotExpired, Data) + retry_inflight_sync(Ref, NotExpired, NData) end. retry_inflight_sync(Ref, QueryOrBatch, Data0) -> @@ -356,7 +386,7 @@ retry_inflight_sync(Ref, QueryOrBatch, Data0) -> ?tp(buffer_worker_retry_inflight, #{query_or_batch => QueryOrBatch, ref => Ref}), QueryOpts = #{simple_query => false}, Result = call_query(force_sync, Id, Index, Ref, QueryOrBatch, QueryOpts), - ReplyResult = + {ShouldAck, PostFn, DeltaCounters} = case QueryOrBatch of ?QUERY(ReplyTo, _, HasBeenSent, _ExpireAt) -> Reply = ?REPLY(ReplyTo, HasBeenSent, Result), @@ -364,9 +394,10 @@ retry_inflight_sync(Ref, QueryOrBatch, Data0) -> [?QUERY(_, _, _, _) | _] = Batch -> batch_reply_caller_defer_metrics(Id, Result, Batch, QueryOpts) end, - case ReplyResult of + Data1 = aggregate_counters(Data0, DeltaCounters), + case ShouldAck of %% Send failed because resource is down - {nack, PostFn} -> + nack -> PostFn(), ?tp( buffer_worker_retry_inflight_failed, @@ -375,11 +406,11 @@ retry_inflight_sync(Ref, QueryOrBatch, Data0) -> query_or_batch => QueryOrBatch } ), - {keep_state, Data0, {state_timeout, ResumeT, unblock}}; + {keep_state, Data1, {state_timeout, ResumeT, unblock}}; %% Send ok or failed but the resource is working - {ack, PostFn} -> + ack -> WorkerPid = self(), - IsAcked = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), + IsAcked = ack_inflight(InflightTID, Ref, WorkerPid), %% we need to defer bumping the counters after %% `inflight_drop' to avoid the race condition when an %% inflight request might get completed concurrently with @@ -394,7 +425,7 @@ retry_inflight_sync(Ref, QueryOrBatch, Data0) -> query_or_batch => QueryOrBatch } ), - resume_from_blocked(Data0) + resume_from_blocked(Data1) end. %% Called during the `running' state only. @@ -426,9 +457,9 @@ collect_and_enqueue_query_requests(Request0, Data0) -> end, Requests ), - {Overflown, NewQ} = append_queue(Id, Index, Q, Queries), + {Overflown, NewQ, DeltaCounters} = append_queue(Id, Index, Q, Queries), ok = reply_overflown(Overflown), - Data0#{queue := NewQ}. + aggregate_counters(Data0#{queue := NewQ}, DeltaCounters). reply_overflown([]) -> ok; @@ -463,8 +494,6 @@ maybe_flush(Data0) -> -spec flush(data()) -> gen_statem:event_handler_result(state(), data()). flush(Data0) -> #{ - id := Id, - index := Index, batch_size := BatchSize, inflight_tid := InflightTID, queue := Q0 @@ -497,13 +526,13 @@ flush(Data0) -> case sieve_expired_requests(Batch, Now) of {[], _AllExpired} -> ok = replayq:ack(Q1, QAckRef), - emqx_resource_metrics:dropped_expired_inc(Id, length(Batch)), - emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), + NumExpired = length(Batch), + Data3 = aggregate_counters(Data2, #{dropped_expired => NumExpired}), ?tp(buffer_worker_flush_all_expired, #{batch => Batch}), - flush(Data2); + flush(Data3); {NotExpired, Expired} -> NumExpired = length(Expired), - emqx_resource_metrics:dropped_expired_inc(Id, NumExpired), + Data3 = aggregate_counters(Data2, #{dropped_expired => NumExpired}), IsBatch = (BatchSize > 1), %% We *must* use the new queue, because we currently can't %% `nack' a `pop'. @@ -513,7 +542,7 @@ flush(Data0) -> #{expired => Expired, not_expired => NotExpired} ), Ref = make_request_ref(), - do_flush(Data2, #{ + do_flush(Data3, #{ is_batch => IsBatch, batch => NotExpired, ref => Ref, @@ -548,7 +577,9 @@ do_flush( QueryOpts = #{inflight_tid => InflightTID, simple_query => false}, Result = call_query(async_if_possible, Id, Index, Ref, Request, QueryOpts), Reply = ?REPLY(ReplyTo, HasBeenSent, Result), - case reply_caller(Id, Reply, QueryOpts) of + {ShouldAck, DeltaCounters} = reply_caller(Id, Reply, QueryOpts), + Data1 = aggregate_counters(Data0, DeltaCounters), + case ShouldAck of %% Failed; remove the request from the queue, as we cannot pop %% from it again, but we'll retry it using the inflight table. nack -> @@ -562,11 +593,10 @@ do_flush( %% request will be retried (i.e., it might not have been %% inserted during `call_query' if the resource was down %% and/or if it was a sync request). - inflight_append(InflightTID, InflightItem, Id, Index), + inflight_append(InflightTID, InflightItem), mark_inflight_as_retriable(InflightTID, Ref), - {Data1, WorkerMRef} = ensure_async_worker_monitored(Data0, Result), + {Data2, WorkerMRef} = ensure_async_worker_monitored(Data1, Result), store_async_worker_reference(InflightTID, Ref, WorkerMRef), - emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), ?tp( buffer_worker_flush_nack, #{ @@ -576,7 +606,7 @@ do_flush( result => Result } ), - {next_state, blocked, Data1}; + {next_state, blocked, Data2}; %% Success; just ack. ack -> ok = replayq:ack(Q1, QAckRef), @@ -588,15 +618,14 @@ do_flush( WorkerPid = self(), case is_async_return(Result) of true when IsUnrecoverableError -> - ack_inflight(InflightTID, Ref, Id, Index, WorkerPid); + ack_inflight(InflightTID, Ref, WorkerPid); true -> ok; false -> - ack_inflight(InflightTID, Ref, Id, Index, WorkerPid) + ack_inflight(InflightTID, Ref, WorkerPid) end, - {Data1, WorkerMRef} = ensure_async_worker_monitored(Data0, Result), + {Data2, WorkerMRef} = ensure_async_worker_monitored(Data1, Result), store_async_worker_reference(InflightTID, Ref, WorkerMRef), - emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), ?tp( buffer_worker_flush_ack, #{ @@ -617,7 +646,7 @@ do_flush( }), ok end, - {keep_state, Data1} + {keep_state, Data2} end; do_flush(#{queue := Q1} = Data0, #{ is_batch := true, @@ -633,7 +662,9 @@ do_flush(#{queue := Q1} = Data0, #{ } = Data0, QueryOpts = #{inflight_tid => InflightTID, simple_query => false}, Result = call_query(async_if_possible, Id, Index, Ref, Batch, QueryOpts), - case batch_reply_caller(Id, Result, Batch, QueryOpts) of + {ShouldAck, DeltaCounters} = batch_reply_caller(Id, Result, Batch, QueryOpts), + Data1 = aggregate_counters(Data0, DeltaCounters), + case ShouldAck of %% Failed; remove the request from the queue, as we cannot pop %% from it again, but we'll retry it using the inflight table. nack -> @@ -647,11 +678,10 @@ do_flush(#{queue := Q1} = Data0, #{ %% request will be retried (i.e., it might not have been %% inserted during `call_query' if the resource was down %% and/or if it was a sync request). - inflight_append(InflightTID, InflightItem, Id, Index), + inflight_append(InflightTID, InflightItem), mark_inflight_as_retriable(InflightTID, Ref), - {Data1, WorkerMRef} = ensure_async_worker_monitored(Data0, Result), + {Data2, WorkerMRef} = ensure_async_worker_monitored(Data1, Result), store_async_worker_reference(InflightTID, Ref, WorkerMRef), - emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), ?tp( buffer_worker_flush_nack, #{ @@ -661,7 +691,7 @@ do_flush(#{queue := Q1} = Data0, #{ result => Result } ), - {next_state, blocked, Data1}; + {next_state, blocked, Data2}; %% Success; just ack. ack -> ok = replayq:ack(Q1, QAckRef), @@ -673,15 +703,14 @@ do_flush(#{queue := Q1} = Data0, #{ WorkerPid = self(), case is_async_return(Result) of true when IsUnrecoverableError -> - ack_inflight(InflightTID, Ref, Id, Index, WorkerPid); + ack_inflight(InflightTID, Ref, WorkerPid); true -> ok; false -> - ack_inflight(InflightTID, Ref, Id, Index, WorkerPid) + ack_inflight(InflightTID, Ref, WorkerPid) end, - {Data1, WorkerMRef} = ensure_async_worker_monitored(Data0, Result), + {Data2, WorkerMRef} = ensure_async_worker_monitored(Data1, Result), store_async_worker_reference(InflightTID, Ref, WorkerMRef), - emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), CurrentCount = queue_count(Q1), ?tp( buffer_worker_flush_ack, @@ -691,13 +720,13 @@ do_flush(#{queue := Q1} = Data0, #{ queue_count => CurrentCount } ), - Data2 = + Data3 = case {CurrentCount > 0, CurrentCount >= BatchSize} of {false, _} -> ?tp_ignore_side_effects_in_prod(buffer_worker_queue_drained, #{ inflight => inflight_count(InflightTID) }), - Data1; + Data2; {true, true} -> ?tp(buffer_worker_flush_ack_reflush, #{ batch_or_query => Batch, @@ -706,17 +735,18 @@ do_flush(#{queue := Q1} = Data0, #{ batch_size => BatchSize }), flush_worker(self()), - Data1; + Data2; {true, false} -> - ensure_flush_timer(Data1) + ensure_flush_timer(Data2) end, - {keep_state, Data2} + {keep_state, Data3} end. batch_reply_caller(Id, BatchResult, Batch, QueryOpts) -> - {ShouldBlock, PostFn} = batch_reply_caller_defer_metrics(Id, BatchResult, Batch, QueryOpts), + {ShouldBlock, PostFn, DeltaCounters} = + batch_reply_caller_defer_metrics(Id, BatchResult, Batch, QueryOpts), PostFn(), - ShouldBlock. + {ShouldBlock, DeltaCounters}. batch_reply_caller_defer_metrics(Id, BatchResult, Batch, QueryOpts) -> %% the `Mod:on_batch_query/3` returns a single result for a batch, @@ -727,23 +757,25 @@ batch_reply_caller_defer_metrics(Id, BatchResult, Batch, QueryOpts) -> end, Batch ), - {ShouldAck, PostFns} = + {ShouldAck, PostFns, Counters} = lists:foldl( - fun(Reply, {_ShouldAck, PostFns}) -> + fun(Reply, {_ShouldAck, PostFns, OldCounters}) -> %% _ShouldAck should be the same as ShouldAck starting from the second reply - {ShouldAck, PostFn} = reply_caller_defer_metrics(Id, Reply, QueryOpts), - {ShouldAck, [PostFn | PostFns]} + {ShouldAck, PostFn, DeltaCounters} = reply_caller_defer_metrics( + Id, Reply, QueryOpts + ), + {ShouldAck, [PostFn | PostFns], merge_counters(OldCounters, DeltaCounters)} end, - {ack, []}, + {ack, [], #{}}, Replies ), PostFn = fun() -> lists:foreach(fun(F) -> F() end, lists:reverse(PostFns)) end, - {ShouldAck, PostFn}. + {ShouldAck, PostFn, Counters}. reply_caller(Id, Reply, QueryOpts) -> - {ShouldAck, PostFn} = reply_caller_defer_metrics(Id, Reply, QueryOpts), + {ShouldAck, PostFn, DeltaCounters} = reply_caller_defer_metrics(Id, Reply, QueryOpts), PostFn(), - ShouldAck. + {ShouldAck, DeltaCounters}. %% Should only reply to the caller when the decision is final (not %% retriable). See comment on `handle_query_result_pure'. @@ -752,7 +784,7 @@ reply_caller_defer_metrics(Id, ?REPLY(undefined, HasBeenSent, Result), _QueryOpt reply_caller_defer_metrics(Id, ?REPLY(ReplyTo, HasBeenSent, Result), QueryOpts) -> IsSimpleQuery = maps:get(simple_query, QueryOpts, false), IsUnrecoverableError = is_unrecoverable_error(Result), - {ShouldAck, PostFn} = handle_query_result_pure(Id, Result, HasBeenSent), + {ShouldAck, PostFn, DeltaCounters} = handle_query_result_pure(Id, Result, HasBeenSent), case {ShouldAck, Result, IsUnrecoverableError, IsSimpleQuery} of {ack, {async_return, _}, true, _} -> ok = do_reply_caller(ReplyTo, Result); @@ -765,11 +797,14 @@ reply_caller_defer_metrics(Id, ?REPLY(ReplyTo, HasBeenSent, Result), QueryOpts) {ack, _, _, _} -> ok = do_reply_caller(ReplyTo, Result) end, - {ShouldAck, PostFn}. + {ShouldAck, PostFn, DeltaCounters}. +%% This is only called by `simple_{,a}sync_query', so we can bump the +%% counters here. handle_query_result(Id, Result, HasBeenSent) -> - {ShouldBlock, PostFn} = handle_query_result_pure(Id, Result, HasBeenSent), + {ShouldBlock, PostFn, DeltaCounters} = handle_query_result_pure(Id, Result, HasBeenSent), PostFn(), + bump_counters(Id, DeltaCounters), ShouldBlock. %% We should always retry (nack), except when: @@ -778,85 +813,156 @@ handle_query_result(Id, Result, HasBeenSent) -> %% * the result is a success (or at least a delayed result) %% We also retry even sync requests. In that case, we shouldn't reply %% the caller until one of those final results above happen. +-spec handle_query_result_pure(id(), term(), HasBeenSent :: boolean()) -> + {ack | nack, function(), counters()}. handle_query_result_pure(_Id, ?RESOURCE_ERROR_M(exception, Msg), _HasBeenSent) -> PostFn = fun() -> ?SLOG(error, #{msg => resource_exception, info => Msg}), ok end, - {nack, PostFn}; + {nack, PostFn, #{}}; handle_query_result_pure(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _HasBeenSent) when NotWorking == not_connected; NotWorking == blocked -> - {nack, fun() -> ok end}; + {nack, fun() -> ok end, #{}}; handle_query_result_pure(Id, ?RESOURCE_ERROR_M(not_found, Msg), _HasBeenSent) -> PostFn = fun() -> ?SLOG(error, #{id => Id, msg => resource_not_found, info => Msg}), - emqx_resource_metrics:dropped_resource_not_found_inc(Id), ok end, - {ack, PostFn}; + {ack, PostFn, #{dropped_resource_not_found => 1}}; handle_query_result_pure(Id, ?RESOURCE_ERROR_M(stopped, Msg), _HasBeenSent) -> PostFn = fun() -> ?SLOG(error, #{id => Id, msg => resource_stopped, info => Msg}), - emqx_resource_metrics:dropped_resource_stopped_inc(Id), ok end, - {ack, PostFn}; + {ack, PostFn, #{dropped_resource_stopped => 1}}; handle_query_result_pure(Id, ?RESOURCE_ERROR_M(Reason, _), _HasBeenSent) -> PostFn = fun() -> ?SLOG(error, #{id => Id, msg => other_resource_error, reason => Reason}), ok end, - {nack, PostFn}; + {nack, PostFn, #{}}; handle_query_result_pure(Id, {error, Reason} = Error, HasBeenSent) -> case is_unrecoverable_error(Error) of true -> PostFn = fun() -> ?SLOG(error, #{id => Id, msg => unrecoverable_error, reason => Reason}), - inc_sent_failed(Id, HasBeenSent), ok end, - {ack, PostFn}; + Counters = + case HasBeenSent of + true -> #{retried_failed => 1}; + false -> #{failed => 1} + end, + {ack, PostFn, Counters}; false -> PostFn = fun() -> ?SLOG(error, #{id => Id, msg => send_error, reason => Reason}), ok end, - {nack, PostFn} + {nack, PostFn, #{}} end; handle_query_result_pure(Id, {async_return, Result}, HasBeenSent) -> handle_query_async_result_pure(Id, Result, HasBeenSent); -handle_query_result_pure(Id, Result, HasBeenSent) -> +handle_query_result_pure(_Id, Result, HasBeenSent) -> PostFn = fun() -> assert_ok_result(Result), - inc_sent_success(Id, HasBeenSent), ok end, - {ack, PostFn}. + Counters = + case HasBeenSent of + true -> #{retried_success => 1}; + false -> #{success => 1} + end, + {ack, PostFn, Counters}. +-spec handle_query_async_result_pure(id(), term(), HasBeenSent :: boolean()) -> + {ack | nack, function(), counters()}. handle_query_async_result_pure(Id, {error, Reason} = Error, HasBeenSent) -> case is_unrecoverable_error(Error) of true -> PostFn = fun() -> ?SLOG(error, #{id => Id, msg => unrecoverable_error, reason => Reason}), - inc_sent_failed(Id, HasBeenSent), ok end, - {ack, PostFn}; + Counters = + case HasBeenSent of + true -> #{retried_failed => 1}; + false -> #{failed => 1} + end, + {ack, PostFn, Counters}; false -> PostFn = fun() -> ?SLOG(error, #{id => Id, msg => async_send_error, reason => Reason}), ok end, - {nack, PostFn} + {nack, PostFn, #{}} end; handle_query_async_result_pure(_Id, {ok, Pid}, _HasBeenSent) when is_pid(Pid) -> - {ack, fun() -> ok end}; + {ack, fun() -> ok end, #{}}; handle_query_async_result_pure(_Id, ok, _HasBeenSent) -> - {ack, fun() -> ok end}. + {ack, fun() -> ok end, #{}}. + +-spec aggregate_counters(data(), counters()) -> data(). +aggregate_counters(Data = #{counters := OldCounters}, DeltaCounters) -> + Counters = merge_counters(OldCounters, DeltaCounters), + Data#{counters := Counters}. + +-spec merge_counters(counters(), counters()) -> counters(). +merge_counters(OldCounters, DeltaCounters) -> + maps:fold( + fun(Metric, Val, Acc) -> + maps:update_with(Metric, fun(X) -> X + Val end, Val, Acc) + end, + OldCounters, + DeltaCounters + ). + +-spec flush_metrics(data()) -> data(). +flush_metrics(Data = #{id := Id, counters := Counters}) -> + bump_counters(Id, Counters), + set_gauges(Data), + ensure_metrics_flush_timer(Data#{counters := #{}}). + +-spec ensure_metrics_flush_timer(data()) -> data(). +ensure_metrics_flush_timer(Data = #{metrics_tref := undefined, metrics_flush_interval := T}) -> + Ref = make_ref(), + TRef = erlang:send_after(T, self(), {flush_metrics, Ref}), + Data#{metrics_tref := {TRef, Ref}}. + +-spec bump_counters(id(), counters()) -> ok. +bump_counters(Id, Counters) -> + maps:foreach( + fun + (dropped_expired, Val) -> + emqx_resource_metrics:dropped_expired_inc(Id, Val); + (dropped_queue_full, Val) -> + emqx_resource_metrics:dropped_queue_full_inc(Id, Val); + (failed, Val) -> + emqx_resource_metrics:failed_inc(Id, Val); + (retried_failed, Val) -> + emqx_resource_metrics:retried_failed_inc(Id, Val); + (success, Val) -> + emqx_resource_metrics:success_inc(Id, Val); + (retried_success, Val) -> + emqx_resource_metrics:retried_success_inc(Id, Val); + (dropped_resource_not_found, Val) -> + emqx_resource_metrics:dropped_resource_not_found_inc(Id, Val); + (dropped_resource_stopped, Val) -> + emqx_resource_metrics:dropped_resource_stopped_inc(Id, Val) + end, + Counters + ). + +-spec set_gauges(data()) -> ok. +set_gauges(_Data = #{id := Id, index := Index, queue := Q, inflight_tid := InflightTID}) -> + emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q)), + emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)), + ok. handle_async_worker_down(Data0, Pid) -> #{async_workers := AsyncWorkers0} = Data0, @@ -942,7 +1048,7 @@ apply_query_fun(async, Mod, Id, Index, Ref, ?QUERY(_, Request, _, _) = Query, Re IsRetriable = false, WorkerMRef = undefined, InflightItem = ?INFLIGHT_ITEM(Ref, Query, IsRetriable, WorkerMRef), - ok = inflight_append(InflightTID, InflightItem, Id, Index), + ok = inflight_append(InflightTID, InflightItem), Result = Mod:on_query_async(Id, Request, {ReplyFun, [ReplyContext]}, ResSt), {async_return, Result} end, @@ -978,7 +1084,7 @@ apply_query_fun(async, Mod, Id, Index, Ref, [?QUERY(_, _, _, _) | _] = Batch, Re IsRetriable = false, WorkerMRef = undefined, InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, WorkerMRef), - ok = inflight_append(InflightTID, InflightItem, Id, Index), + ok = inflight_append(InflightTID, InflightItem), Result = Mod:on_batch_query_async(Id, Requests, {ReplyFun, [ReplyContext]}, ResSt), {async_return, Result} end, @@ -1005,7 +1111,6 @@ handle_async_reply1( request_ref := Ref, inflight_tid := InflightTID, resource_id := Id, - worker_index := Index, buffer_worker := WorkerPid, min_query := ?QUERY(_, _, _, ExpireAt) = _Query } = ReplyContext, @@ -1018,7 +1123,9 @@ handle_async_reply1( Now = now_(), case is_expired(ExpireAt, Now) of true -> - IsAcked = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), + IsAcked = ack_inflight(InflightTID, Ref, WorkerPid), + %% evalutate metrics call here since we're not inside + %% buffer worker IsAcked andalso emqx_resource_metrics:late_reply_inc(Id), ?tp(handle_async_reply_expired, #{expired => [_Query]}), ok; @@ -1031,7 +1138,6 @@ do_handle_async_reply( query_opts := QueryOpts, resource_id := Id, request_ref := Ref, - worker_index := Index, buffer_worker := WorkerPid, inflight_tid := InflightTID, min_query := ?QUERY(ReplyTo, _, Sent, _ExpireAt) = _Query @@ -1041,7 +1147,7 @@ do_handle_async_reply( %% NOTE: 'inflight' is the count of messages that were sent async %% but received no ACK, NOT the number of messages queued in the %% inflight window. - {Action, PostFn} = reply_caller_defer_metrics( + {Action, PostFn, DeltaCounters} = reply_caller_defer_metrics( Id, ?REPLY(ReplyTo, Sent, Result), QueryOpts ), @@ -1058,7 +1164,7 @@ do_handle_async_reply( ok = ?MODULE:block(WorkerPid), blocked; ack -> - ok = do_async_ack(InflightTID, Ref, Id, Index, WorkerPid, PostFn, QueryOpts) + ok = do_async_ack(InflightTID, Ref, Id, PostFn, WorkerPid, DeltaCounters, QueryOpts) end. handle_async_batch_reply( @@ -1110,7 +1216,6 @@ handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> ?INFLIGHT_ITEM(_, RealBatch, _IsRetriable, _WorkerMRef) = Inflight, #{ resource_id := Id, - worker_index := Index, buffer_worker := WorkerPid, inflight_tid := InflightTID, request_ref := Ref, @@ -1130,11 +1235,13 @@ handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> RealNotExpired0 ), NumExpired = length(RealExpired), + %% evalutate metrics call here since we're not inside buffer + %% worker emqx_resource_metrics:late_reply_inc(Id, NumExpired), case RealNotExpired of [] -> %% all expired, no need to update back the inflight batch - _ = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), + _ = ack_inflight(InflightTID, Ref, WorkerPid), ok; _ -> %% some queries are not expired, put them back to the inflight batch @@ -1147,7 +1254,6 @@ do_handle_async_batch_reply( #{ buffer_worker := WorkerPid, resource_id := Id, - worker_index := Index, inflight_tid := InflightTID, request_ref := Ref, min_batch := Batch, @@ -1155,7 +1261,9 @@ do_handle_async_batch_reply( }, Result ) -> - {Action, PostFn} = batch_reply_caller_defer_metrics(Id, Result, Batch, QueryOpts), + {Action, PostFn, DeltaCounters} = batch_reply_caller_defer_metrics( + Id, Result, Batch, QueryOpts + ), ?tp(handle_async_reply, #{ action => Action, batch_or_query => Batch, @@ -1169,16 +1277,18 @@ do_handle_async_batch_reply( ok = ?MODULE:block(WorkerPid), blocked; ack -> - ok = do_async_ack(InflightTID, Ref, Id, Index, WorkerPid, PostFn, QueryOpts) + ok = do_async_ack(InflightTID, Ref, Id, PostFn, WorkerPid, DeltaCounters, QueryOpts) end. -do_async_ack(InflightTID, Ref, Id, Index, WorkerPid, PostFn, QueryOpts) -> - IsKnownRef = ack_inflight(InflightTID, Ref, Id, Index, WorkerPid), +do_async_ack(InflightTID, Ref, Id, PostFn, WorkerPid, DeltaCounters, QueryOpts) -> + IsKnownRef = ack_inflight(InflightTID, Ref, WorkerPid), case maps:get(simple_query, QueryOpts, false) of true -> - PostFn(); + PostFn(), + bump_counters(Id, DeltaCounters); false when IsKnownRef -> - PostFn(); + PostFn(), + bump_counters(Id, DeltaCounters); false -> ok end, @@ -1222,31 +1332,30 @@ estimate_size(QItem) -> erlang:external_size(QItem). -spec append_queue(id(), index(), replayq:q(), [queue_query()]) -> - {[queue_query()], replayq:q()}. + {[queue_query()], replayq:q(), counters()}. append_queue(Id, Index, Q, Queries) -> %% this assertion is to ensure that we never append a raw binary %% because the marshaller will get lost. false = is_binary(hd(Queries)), Q0 = replayq:append(Q, Queries), - {Overflown, Q2} = + {Overflown, Q2, DeltaCounters} = case replayq:overflow(Q0) of OverflownBytes when OverflownBytes =< 0 -> - {[], Q0}; + {[], Q0, #{}}; OverflownBytes -> PopOpts = #{bytes_limit => OverflownBytes, count_limit => 999999999}, {Q1, QAckRef, Items2} = replayq:pop(Q0, PopOpts), ok = replayq:ack(Q1, QAckRef), Dropped = length(Items2), - emqx_resource_metrics:dropped_queue_full_inc(Id, Dropped), + Counters = #{dropped_queue_full => Dropped}, ?SLOG(info, #{ msg => buffer_worker_overflow, resource_id => Id, worker_index => Index, dropped => Dropped }), - {Items2, Q1} + {Items2, Q1, Counters} end, - emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q2)), ?tp( buffer_worker_appended_to_queue, #{ @@ -1256,7 +1365,7 @@ append_queue(Id, Index, Q, Queries) -> overflown => length(Overflown) } ), - {Overflown, Q2}. + {Overflown, Q2, DeltaCounters}. %%============================================================================== %% the inflight queue for async query @@ -1266,20 +1375,18 @@ append_queue(Id, Index, Q, Queries) -> -define(INITIAL_TIME_REF, initial_time). -define(INITIAL_MONOTONIC_TIME_REF, initial_monotonic_time). -inflight_new(InfltWinSZ, Id, Index) -> +inflight_new(InfltWinSZ) -> TableId = ets:new( emqx_resource_buffer_worker_inflight_tab, [ordered_set, public, {write_concurrency, true}] ), - inflight_append(TableId, {?MAX_SIZE_REF, InfltWinSZ}, Id, Index), + inflight_append(TableId, {?MAX_SIZE_REF, InfltWinSZ}), %% we use this counter because we might deal with batches as %% elements. - inflight_append(TableId, {?SIZE_REF, 0}, Id, Index), - inflight_append(TableId, {?BATCH_COUNT_REF, 0}, Id, Index), - inflight_append(TableId, {?INITIAL_TIME_REF, erlang:system_time()}, Id, Index), - inflight_append( - TableId, {?INITIAL_MONOTONIC_TIME_REF, make_request_ref()}, Id, Index - ), + inflight_append(TableId, {?SIZE_REF, 0}), + inflight_append(TableId, {?BATCH_COUNT_REF, 0}), + inflight_append(TableId, {?INITIAL_TIME_REF, erlang:system_time()}), + inflight_append(TableId, {?INITIAL_MONOTONIC_TIME_REF, make_request_ref()}), TableId. -spec inflight_get_first_retriable(ets:tid(), integer()) -> @@ -1331,38 +1438,32 @@ inflight_num_msgs(InflightTID) -> [{_, Size}] = ets:lookup(InflightTID, ?SIZE_REF), Size. -inflight_append(undefined, _InflightItem, _Id, _Index) -> +inflight_append(undefined, _InflightItem) -> ok; inflight_append( InflightTID, - ?INFLIGHT_ITEM(Ref, [?QUERY(_, _, _, _) | _] = Batch0, IsRetriable, WorkerMRef), - Id, - Index + ?INFLIGHT_ITEM(Ref, [?QUERY(_, _, _, _) | _] = Batch0, IsRetriable, WorkerMRef) ) -> Batch = mark_as_sent(Batch0), InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, WorkerMRef), IsNew = ets:insert_new(InflightTID, InflightItem), BatchSize = length(Batch), IsNew andalso inc_inflight(InflightTID, BatchSize), - emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)), ?tp(buffer_worker_appended_to_inflight, #{item => InflightItem, is_new => IsNew}), ok; inflight_append( InflightTID, ?INFLIGHT_ITEM( Ref, ?QUERY(_ReplyTo, _Req, _HasBeenSent, _ExpireAt) = Query0, IsRetriable, WorkerMRef - ), - Id, - Index + ) ) -> Query = mark_as_sent(Query0), InflightItem = ?INFLIGHT_ITEM(Ref, Query, IsRetriable, WorkerMRef), IsNew = ets:insert_new(InflightTID, InflightItem), IsNew andalso inc_inflight(InflightTID, 1), - emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)), ?tp(buffer_worker_appended_to_inflight, #{item => InflightItem, is_new => IsNew}), ok; -inflight_append(InflightTID, {Ref, Data}, _Id, _Index) -> +inflight_append(InflightTID, {Ref, Data}) -> ets:insert(InflightTID, {Ref, Data}), %% this is a metadata row being inserted; therefore, we don't bump %% the inflight metric. @@ -1398,6 +1499,8 @@ ensure_async_worker_monitored( ensure_async_worker_monitored(Data0, _Result) -> {Data0, undefined}. +-spec store_async_worker_reference(undefined | ets:tid(), inflight_key(), undefined | reference()) -> + ok. store_async_worker_reference(undefined = _InflightTID, _Ref, _WorkerMRef) -> ok; store_async_worker_reference(_InflightTID, _Ref, undefined = _WorkerRef) -> @@ -1410,9 +1513,9 @@ store_async_worker_reference(InflightTID, Ref, WorkerMRef) when ), ok. -ack_inflight(undefined, _Ref, _Id, _Index, _WorkerPid) -> +ack_inflight(undefined, _Ref, _WorkerPid) -> false; -ack_inflight(InflightTID, Ref, Id, Index, WorkerPid) -> +ack_inflight(InflightTID, Ref, WorkerPid) -> {Count, Removed} = case ets:take(InflightTID, Ref) of [?INFLIGHT_ITEM(Ref, ?QUERY(_, _, _, _), _IsRetriable, _WorkerMRef)] -> @@ -1428,12 +1531,6 @@ ack_inflight(InflightTID, Ref, Id, Index, WorkerPid) -> flush -> ?MODULE:flush_worker(WorkerPid) end, IsKnownRef = (Count > 0), - case IsKnownRef of - true -> - emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)); - false -> - ok - end, IsKnownRef. mark_inflight_items_as_retriable(Data, WorkerMRef) -> @@ -1496,16 +1593,6 @@ dec_inflight_update(InflightTID, Count) when Count > 0 -> %%============================================================================== -inc_sent_failed(Id, _HasBeenSent = true) -> - emqx_resource_metrics:retried_failed_inc(Id); -inc_sent_failed(Id, _HasBeenSent) -> - emqx_resource_metrics:failed_inc(Id). - -inc_sent_success(Id, _HasBeenSent = true) -> - emqx_resource_metrics:retried_success_inc(Id); -inc_sent_success(Id, _HasBeenSent) -> - emqx_resource_metrics:success_inc(Id). - call_mode(force_sync, _) -> sync; call_mode(async_if_possible, always_sync) -> sync; call_mode(async_if_possible, async_if_possible) -> async. diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 3b4fb66e5..1de1a6545 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -44,6 +44,7 @@ fields("creation_opts") -> {worker_pool_size, fun worker_pool_size/1}, {health_check_interval, fun health_check_interval/1}, {resume_interval, fun resume_interval/1}, + {metrics_flush_interval, fun metrics_flush_interval/1}, {start_after_created, fun start_after_created/1}, {start_timeout, fun start_timeout/1}, {auto_restart_interval, fun auto_restart_interval/1}, @@ -77,6 +78,11 @@ resume_interval(desc) -> ?DESC("resume_interval"); resume_interval(required) -> false; resume_interval(_) -> undefined. +metrics_flush_interval(type) -> emqx_schema:duration_ms(); +metrics_flush_interval(importance) -> ?IMPORTANCE_HIDDEN; +metrics_flush_interval(required) -> false; +metrics_flush_interval(_) -> undefined. + health_check_interval(type) -> emqx_schema:duration_ms(); health_check_interval(desc) -> ?DESC("health_check_interval"); health_check_interval(default) -> ?HEALTHCHECK_INTERVAL_RAW; diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index fc338b512..b960b0526 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -316,7 +316,11 @@ t_query_counter_async_query(_) -> ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, #{name => test_resource, register => true}, - #{query_mode => async, batch_size => 1} + #{ + query_mode => async, + batch_size => 1, + metrics_flush_interval => 50 + } ), ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)), NMsgs = 1_000, @@ -350,7 +354,11 @@ t_query_counter_async_query(_) -> end ), #{counters := C} = emqx_resource:get_metrics(?ID), - ?assertMatch(#{matched := 1002, 'success' := 1002, 'failed' := 0}, C), + ?retry( + _Sleep = 300, + _Attempts0 = 20, + ?assertMatch(#{matched := 1002, 'success' := 1002, 'failed' := 0}, C) + ), ok = emqx_resource:remove_local(?ID). t_query_counter_async_callback(_) -> @@ -1171,6 +1179,7 @@ t_unblock_only_required_buffer_workers(_) -> #{ query_mode => async, batch_size => 5, + metrics_flush_interval => 50, batch_time => 100 } ), @@ -1219,6 +1228,7 @@ t_retry_batch(_Config) -> batch_size => 5, batch_time => 100, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => 1_000 } ), @@ -1318,6 +1328,7 @@ t_delete_and_re_create_with_same_name(_Config) -> worker_pool_size => NumBufferWorkers, buffer_mode => volatile_offload, buffer_seg_bytes => 100, + metrics_flush_interval => 50, resume_interval => 1_000 } ), @@ -1354,10 +1365,16 @@ t_delete_and_re_create_with_same_name(_Config) -> %% ensure that stuff got enqueued into disk tap_metrics(?LINE), - Queuing1 = emqx_resource_metrics:queuing_get(?ID), - Inflight1 = emqx_resource_metrics:inflight_get(?ID), - ?assert(Queuing1 > 0), - ?assertEqual(2, Inflight1), + ?retry( + _Sleep = 300, + _Attempts0 = 20, + ?assert(emqx_resource_metrics:queuing_get(?ID) > 0) + ), + ?retry( + _Sleep = 300, + _Attempts0 = 20, + ?assertEqual(2, emqx_resource_metrics:inflight_get(?ID)) + ), %% now, we delete the resource process_flag(trap_exit, true), @@ -1409,6 +1426,7 @@ t_always_overflow(_Config) -> batch_size => 1, worker_pool_size => 1, max_buffer_bytes => 1, + metrics_flush_interval => 50, resume_interval => 1_000 } ), @@ -1446,6 +1464,7 @@ t_retry_sync_inflight(_Config) -> query_mode => sync, batch_size => 1, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => ResumeInterval } ), @@ -1496,6 +1515,7 @@ t_retry_sync_inflight_batch(_Config) -> batch_size => 2, batch_time => 200, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => ResumeInterval } ), @@ -1546,6 +1566,7 @@ t_retry_async_inflight(_Config) -> query_mode => async, batch_size => 1, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => ResumeInterval } ), @@ -1590,6 +1611,7 @@ t_retry_async_inflight_full(_Config) -> inflight_window => AsyncInflightWindow, batch_size => 1, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => ResumeInterval } ), @@ -1653,6 +1675,7 @@ t_async_reply_multi_eval(_Config) -> batch_size => 3, batch_time => 10, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => ResumeInterval } ), @@ -1667,7 +1690,7 @@ t_async_reply_multi_eval(_Config) -> #{} ), ?retry( - ResumeInterval, + 2 * ResumeInterval, TotalTime div ResumeInterval, begin Metrics = tap_metrics(?LINE), @@ -1683,7 +1706,7 @@ t_async_reply_multi_eval(_Config) -> failed := Failed } = Counters, ?assertEqual(TotalQueries, Matched - 1), - ?assertEqual(Matched, Success + Dropped + LateReply + Failed) + ?assertEqual(Matched, Success + Dropped + LateReply + Failed, #{counters => Counters}) end ). @@ -1700,6 +1723,7 @@ t_retry_async_inflight_batch(_Config) -> batch_size => 2, batch_time => 200, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => ResumeInterval } ), @@ -1745,6 +1769,7 @@ t_async_pool_worker_death(_Config) -> query_mode => async, batch_size => 1, worker_pool_size => NumBufferWorkers, + metrics_refresh_interval => 50, resume_interval => ResumeInterval } ), @@ -1768,8 +1793,11 @@ t_async_pool_worker_death(_Config) -> inc_counter_in_parallel_increasing(NumReqs, 1, ReqOpts), {ok, _} = snabbkaffe:receive_events(SRef0), - Inflight0 = emqx_resource_metrics:inflight_get(?ID), - ?assertEqual(NumReqs, Inflight0), + ?retry( + _Sleep = 300, + _Attempts0 = 20, + ?assertEqual(NumReqs, emqx_resource_metrics:inflight_get(?ID)) + ), %% grab one of the worker pids and kill it {ok, #{pid := Pid0}} = emqx_resource:simple_sync_query(?ID, get_state), @@ -1820,6 +1848,7 @@ t_expiration_sync_before_sending(_Config) -> query_mode => sync, batch_size => 1, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => 1_000 } ), @@ -1837,6 +1866,7 @@ t_expiration_sync_batch_before_sending(_Config) -> batch_size => 2, batch_time => 100, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => 1_000 } ), @@ -1853,6 +1883,7 @@ t_expiration_async_before_sending(_Config) -> query_mode => async, batch_size => 1, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => 1_000 } ), @@ -1870,6 +1901,7 @@ t_expiration_async_batch_before_sending(_Config) -> batch_size => 2, batch_time => 100, worker_pool_size => 1, + metrics_flush_interval => 50, resume_interval => 1_000 } ), @@ -1950,6 +1982,7 @@ t_expiration_sync_before_sending_partial_batch(_Config) -> batch_size => 2, batch_time => 100, worker_pool_size => 1, + metrics_flush_interval => 250, resume_interval => 1_000 } ), @@ -1968,6 +2001,7 @@ t_expiration_async_before_sending_partial_batch(_Config) -> batch_size => 2, batch_time => 100, worker_pool_size => 1, + metrics_flush_interval => 250, resume_interval => 1_000 } ), @@ -2057,7 +2091,14 @@ do_t_expiration_before_sending_partial_batch(QueryMode) -> ], ?of_kind(buffer_worker_flush_potentially_partial, Trace) ), - wait_until_gauge_is(inflight, 0, 500), + wait_until_gauge_is( + inflight, + #{ + expected_value => 0, + timeout => 500, + max_events => 10 + } + ), Metrics = tap_metrics(?LINE), case QueryMode of async -> @@ -2933,8 +2974,15 @@ install_telemetry_handler(TestCase) -> put({?MODULE, telemetry_table}, Tid), Tid. -wait_until_gauge_is(GaugeName, ExpectedValue, Timeout) -> - Events = receive_all_events(GaugeName, Timeout), +wait_until_gauge_is( + GaugeName, + #{ + expected_value := ExpectedValue, + timeout := Timeout, + max_events := MaxEvents + } +) -> + Events = receive_all_events(GaugeName, Timeout, MaxEvents), case length(Events) > 0 andalso lists:last(Events) of #{measurements := #{gauge_set := ExpectedValue}} -> ok; @@ -2948,12 +2996,18 @@ wait_until_gauge_is(GaugeName, ExpectedValue, Timeout) -> end. receive_all_events(EventName, Timeout) -> - receive_all_events(EventName, Timeout, []). + receive_all_events(EventName, Timeout, _MaxEvents = 50, _Count = 0, _Acc = []). -receive_all_events(EventName, Timeout, Acc) -> +receive_all_events(EventName, Timeout, MaxEvents) -> + receive_all_events(EventName, Timeout, MaxEvents, _Count = 0, _Acc = []). + +receive_all_events(_EventName, _Timeout, MaxEvents, Count, Acc) when Count >= MaxEvents -> + lists:reverse(Acc); +receive_all_events(EventName, Timeout, MaxEvents, Count, Acc) -> receive {telemetry, #{name := [_, _, EventName]} = Event} -> - receive_all_events(EventName, Timeout, [Event | Acc]) + ct:pal("telemetry event: ~p", [Event]), + receive_all_events(EventName, Timeout, MaxEvents, Count + 1, [Event | Acc]) after Timeout -> lists:reverse(Acc) end. From c74c93388e5ed440a705f25d85ac2bf101c41b18 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 18 May 2023 09:48:25 -0300 Subject: [PATCH 138/144] refactor: rename some variables and sum type constructors for clarity --- .../src/emqx_resource_buffer_worker.erl | 154 +++++++++--------- 1 file changed, 79 insertions(+), 75 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 993e69749..47769418b 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -63,8 +63,8 @@ -define(QUERY(FROM, REQUEST, SENT, EXPIRE_AT), {query, FROM, REQUEST, SENT, EXPIRE_AT}). -define(SIMPLE_QUERY(REQUEST), ?QUERY(undefined, REQUEST, false, infinity)). -define(REPLY(FROM, SENT, RESULT), {reply, FROM, SENT, RESULT}). --define(INFLIGHT_ITEM(Ref, BatchOrQuery, IsRetriable, WorkerMRef), - {Ref, BatchOrQuery, IsRetriable, WorkerMRef} +-define(INFLIGHT_ITEM(Ref, BatchOrQuery, IsRetriable, AsyncWorkerMRef), + {Ref, BatchOrQuery, IsRetriable, AsyncWorkerMRef} ). -define(ITEM_IDX, 2). -define(RETRY_IDX, 3). @@ -350,8 +350,8 @@ resume_from_blocked(Data) -> {next_state, running, Data} end; {expired, Ref, Batch} -> - WorkerPid = self(), - IsAcked = ack_inflight(InflightTID, Ref, WorkerPid), + BufferWorkerPid = self(), + IsAcked = ack_inflight(InflightTID, Ref, BufferWorkerPid), Counters = case IsAcked of true -> #{dropped_expired => length(Batch)}; @@ -409,8 +409,8 @@ retry_inflight_sync(Ref, QueryOrBatch, Data0) -> {keep_state, Data1, {state_timeout, ResumeT, unblock}}; %% Send ok or failed but the resource is working ack -> - WorkerPid = self(), - IsAcked = ack_inflight(InflightTID, Ref, WorkerPid), + BufferWorkerPid = self(), + IsAcked = ack_inflight(InflightTID, Ref, BufferWorkerPid), %% we need to defer bumping the counters after %% `inflight_drop' to avoid the race condition when an %% inflight request might get completed concurrently with @@ -587,16 +587,16 @@ do_flush( %% we set it atomically just below; a limitation of having %% to use tuples for atomic ets updates IsRetriable = true, - WorkerMRef0 = undefined, - InflightItem = ?INFLIGHT_ITEM(Ref, Request, IsRetriable, WorkerMRef0), + AsyncWorkerMRef0 = undefined, + InflightItem = ?INFLIGHT_ITEM(Ref, Request, IsRetriable, AsyncWorkerMRef0), %% we must append again to the table to ensure that the %% request will be retried (i.e., it might not have been %% inserted during `call_query' if the resource was down %% and/or if it was a sync request). inflight_append(InflightTID, InflightItem), mark_inflight_as_retriable(InflightTID, Ref), - {Data2, WorkerMRef} = ensure_async_worker_monitored(Data1, Result), - store_async_worker_reference(InflightTID, Ref, WorkerMRef), + {Data2, AsyncWorkerMRef} = ensure_async_worker_monitored(Data1, Result), + store_async_worker_reference(InflightTID, Ref, AsyncWorkerMRef), ?tp( buffer_worker_flush_nack, #{ @@ -615,17 +615,17 @@ do_flush( %% must ensure the async worker is being monitored for %% such requests. IsUnrecoverableError = is_unrecoverable_error(Result), - WorkerPid = self(), + BufferWorkerPid = self(), case is_async_return(Result) of true when IsUnrecoverableError -> - ack_inflight(InflightTID, Ref, WorkerPid); + ack_inflight(InflightTID, Ref, BufferWorkerPid); true -> ok; false -> - ack_inflight(InflightTID, Ref, WorkerPid) + ack_inflight(InflightTID, Ref, BufferWorkerPid) end, - {Data2, WorkerMRef} = ensure_async_worker_monitored(Data1, Result), - store_async_worker_reference(InflightTID, Ref, WorkerMRef), + {Data2, AsyncWorkerMRef} = ensure_async_worker_monitored(Data1, Result), + store_async_worker_reference(InflightTID, Ref, AsyncWorkerMRef), ?tp( buffer_worker_flush_ack, #{ @@ -672,16 +672,16 @@ do_flush(#{queue := Q1} = Data0, #{ %% we set it atomically just below; a limitation of having %% to use tuples for atomic ets updates IsRetriable = true, - WorkerMRef0 = undefined, - InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, WorkerMRef0), + AsyncWorkerMRef0 = undefined, + InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, AsyncWorkerMRef0), %% we must append again to the table to ensure that the %% request will be retried (i.e., it might not have been %% inserted during `call_query' if the resource was down %% and/or if it was a sync request). inflight_append(InflightTID, InflightItem), mark_inflight_as_retriable(InflightTID, Ref), - {Data2, WorkerMRef} = ensure_async_worker_monitored(Data1, Result), - store_async_worker_reference(InflightTID, Ref, WorkerMRef), + {Data2, AsyncWorkerMRef} = ensure_async_worker_monitored(Data1, Result), + store_async_worker_reference(InflightTID, Ref, AsyncWorkerMRef), ?tp( buffer_worker_flush_nack, #{ @@ -700,17 +700,17 @@ do_flush(#{queue := Q1} = Data0, #{ %% must ensure the async worker is being monitored for %% such requests. IsUnrecoverableError = is_unrecoverable_error(Result), - WorkerPid = self(), + BufferWorkerPid = self(), case is_async_return(Result) of true when IsUnrecoverableError -> - ack_inflight(InflightTID, Ref, WorkerPid); + ack_inflight(InflightTID, Ref, BufferWorkerPid); true -> ok; false -> - ack_inflight(InflightTID, Ref, WorkerPid) + ack_inflight(InflightTID, Ref, BufferWorkerPid) end, - {Data2, WorkerMRef} = ensure_async_worker_monitored(Data1, Result), - store_async_worker_reference(InflightTID, Ref, WorkerMRef), + {Data2, AsyncWorkerMRef} = ensure_async_worker_monitored(Data1, Result), + store_async_worker_reference(InflightTID, Ref, AsyncWorkerMRef), CurrentCount = queue_count(Q1), ?tp( buffer_worker_flush_ack, @@ -966,9 +966,9 @@ set_gauges(_Data = #{id := Id, index := Index, queue := Q, inflight_tid := Infli handle_async_worker_down(Data0, Pid) -> #{async_workers := AsyncWorkers0} = Data0, - {WorkerMRef, AsyncWorkers} = maps:take(Pid, AsyncWorkers0), + {AsyncWorkerMRef, AsyncWorkers} = maps:take(Pid, AsyncWorkers0), Data = Data0#{async_workers := AsyncWorkers}, - mark_inflight_items_as_retriable(Data, WorkerMRef), + mark_inflight_items_as_retriable(Data, AsyncWorkerMRef), {keep_state, Data}. -spec call_query(force_sync | async_if_possible, _, _, _, _, _) -> _. @@ -1046,8 +1046,8 @@ apply_query_fun(async, Mod, Id, Index, Ref, ?QUERY(_, Request, _, _) = Query, Re min_query => minimize(Query) }, IsRetriable = false, - WorkerMRef = undefined, - InflightItem = ?INFLIGHT_ITEM(Ref, Query, IsRetriable, WorkerMRef), + AsyncWorkerMRef = undefined, + InflightItem = ?INFLIGHT_ITEM(Ref, Query, IsRetriable, AsyncWorkerMRef), ok = inflight_append(InflightTID, InflightItem), Result = Mod:on_query_async(Id, Request, {ReplyFun, [ReplyContext]}, ResSt), {async_return, Result} @@ -1082,8 +1082,8 @@ apply_query_fun(async, Mod, Id, Index, Ref, [?QUERY(_, _, _, _) | _] = Batch, Re fun(?QUERY(_ReplyTo, Request, _, _ExpireAt)) -> Request end, Batch ), IsRetriable = false, - WorkerMRef = undefined, - InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, WorkerMRef), + AsyncWorkerMRef = undefined, + InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, AsyncWorkerMRef), ok = inflight_append(InflightTID, InflightItem), Result = Mod:on_batch_query_async(Id, Requests, {ReplyFun, [ReplyContext]}, ResSt), {async_return, Result} @@ -1111,7 +1111,7 @@ handle_async_reply1( request_ref := Ref, inflight_tid := InflightTID, resource_id := Id, - buffer_worker := WorkerPid, + buffer_worker := BufferWorkerPid, min_query := ?QUERY(_, _, _, ExpireAt) = _Query } = ReplyContext, Result @@ -1123,7 +1123,7 @@ handle_async_reply1( Now = now_(), case is_expired(ExpireAt, Now) of true -> - IsAcked = ack_inflight(InflightTID, Ref, WorkerPid), + IsAcked = ack_inflight(InflightTID, Ref, BufferWorkerPid), %% evalutate metrics call here since we're not inside %% buffer worker IsAcked andalso emqx_resource_metrics:late_reply_inc(Id), @@ -1138,7 +1138,7 @@ do_handle_async_reply( query_opts := QueryOpts, resource_id := Id, request_ref := Ref, - buffer_worker := WorkerPid, + buffer_worker := BufferWorkerPid, inflight_tid := InflightTID, min_query := ?QUERY(ReplyTo, _, Sent, _ExpireAt) = _Query }, @@ -1161,10 +1161,12 @@ do_handle_async_reply( nack -> %% Keep retrying. ok = mark_inflight_as_retriable(InflightTID, Ref), - ok = ?MODULE:block(WorkerPid), + ok = ?MODULE:block(BufferWorkerPid), blocked; ack -> - ok = do_async_ack(InflightTID, Ref, Id, PostFn, WorkerPid, DeltaCounters, QueryOpts) + ok = do_async_ack( + InflightTID, Ref, Id, PostFn, BufferWorkerPid, DeltaCounters, QueryOpts + ) end. handle_async_batch_reply( @@ -1213,10 +1215,10 @@ handle_async_batch_reply2([], _, _, _) -> %% this usually should never happen unless the async callback is being evaluated concurrently ok; handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> - ?INFLIGHT_ITEM(_, RealBatch, _IsRetriable, _WorkerMRef) = Inflight, + ?INFLIGHT_ITEM(_, RealBatch, _IsRetriable, _AsyncWorkerMRef) = Inflight, #{ resource_id := Id, - buffer_worker := WorkerPid, + buffer_worker := BufferWorkerPid, inflight_tid := InflightTID, request_ref := Ref, min_batch := Batch @@ -1241,7 +1243,7 @@ handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> case RealNotExpired of [] -> %% all expired, no need to update back the inflight batch - _ = ack_inflight(InflightTID, Ref, WorkerPid), + _ = ack_inflight(InflightTID, Ref, BufferWorkerPid), ok; _ -> %% some queries are not expired, put them back to the inflight batch @@ -1252,7 +1254,7 @@ handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> do_handle_async_batch_reply( #{ - buffer_worker := WorkerPid, + buffer_worker := BufferWorkerPid, resource_id := Id, inflight_tid := InflightTID, request_ref := Ref, @@ -1274,14 +1276,16 @@ do_handle_async_batch_reply( nack -> %% Keep retrying. ok = mark_inflight_as_retriable(InflightTID, Ref), - ok = ?MODULE:block(WorkerPid), + ok = ?MODULE:block(BufferWorkerPid), blocked; ack -> - ok = do_async_ack(InflightTID, Ref, Id, PostFn, WorkerPid, DeltaCounters, QueryOpts) + ok = do_async_ack( + InflightTID, Ref, Id, PostFn, BufferWorkerPid, DeltaCounters, QueryOpts + ) end. -do_async_ack(InflightTID, Ref, Id, PostFn, WorkerPid, DeltaCounters, QueryOpts) -> - IsKnownRef = ack_inflight(InflightTID, Ref, WorkerPid), +do_async_ack(InflightTID, Ref, Id, PostFn, BufferWorkerPid, DeltaCounters, QueryOpts) -> + IsKnownRef = ack_inflight(InflightTID, Ref, BufferWorkerPid), case maps:get(simple_query, QueryOpts, false) of true -> PostFn(), @@ -1397,7 +1401,7 @@ inflight_new(InfltWinSZ) -> inflight_get_first_retriable(InflightTID, Now) -> MatchSpec = ets:fun2ms( - fun(?INFLIGHT_ITEM(Ref, BatchOrQuery, IsRetriable, _WorkerMRef)) when + fun(?INFLIGHT_ITEM(Ref, BatchOrQuery, IsRetriable, _AsyncWorkerMRef)) when IsRetriable =:= true -> {Ref, BatchOrQuery} @@ -1442,10 +1446,10 @@ inflight_append(undefined, _InflightItem) -> ok; inflight_append( InflightTID, - ?INFLIGHT_ITEM(Ref, [?QUERY(_, _, _, _) | _] = Batch0, IsRetriable, WorkerMRef) + ?INFLIGHT_ITEM(Ref, [?QUERY(_, _, _, _) | _] = Batch0, IsRetriable, AsyncWorkerMRef) ) -> Batch = mark_as_sent(Batch0), - InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, WorkerMRef), + InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, AsyncWorkerMRef), IsNew = ets:insert_new(InflightTID, InflightItem), BatchSize = length(Batch), IsNew andalso inc_inflight(InflightTID, BatchSize), @@ -1454,11 +1458,11 @@ inflight_append( inflight_append( InflightTID, ?INFLIGHT_ITEM( - Ref, ?QUERY(_ReplyTo, _Req, _HasBeenSent, _ExpireAt) = Query0, IsRetriable, WorkerMRef + Ref, ?QUERY(_ReplyTo, _Req, _HasBeenSent, _ExpireAt) = Query0, IsRetriable, AsyncWorkerMRef ) ) -> Query = mark_as_sent(Query0), - InflightItem = ?INFLIGHT_ITEM(Ref, Query, IsRetriable, WorkerMRef), + InflightItem = ?INFLIGHT_ITEM(Ref, Query, IsRetriable, AsyncWorkerMRef), IsNew = ets:insert_new(InflightTID, InflightItem), IsNew andalso inc_inflight(InflightTID, 1), ?tp(buffer_worker_appended_to_inflight, #{item => InflightItem, is_new => IsNew}), @@ -1481,67 +1485,67 @@ mark_inflight_as_retriable(InflightTID, Ref) -> %% Track each worker pid only once. ensure_async_worker_monitored( - Data0 = #{async_workers := AsyncWorkers}, {async_return, {ok, WorkerPid}} = _Result + Data0 = #{async_workers := AsyncWorkers}, {async_return, {ok, AsyncWorkerPid}} = _Result ) when - is_pid(WorkerPid), is_map_key(WorkerPid, AsyncWorkers) + is_pid(AsyncWorkerPid), is_map_key(AsyncWorkerPid, AsyncWorkers) -> - WorkerMRef = maps:get(WorkerPid, AsyncWorkers), - {Data0, WorkerMRef}; + AsyncWorkerMRef = maps:get(AsyncWorkerPid, AsyncWorkers), + {Data0, AsyncWorkerMRef}; ensure_async_worker_monitored( - Data0 = #{async_workers := AsyncWorkers0}, {async_return, {ok, WorkerPid}} + Data0 = #{async_workers := AsyncWorkers0}, {async_return, {ok, AsyncWorkerPid}} ) when - is_pid(WorkerPid) + is_pid(AsyncWorkerPid) -> - WorkerMRef = monitor(process, WorkerPid), - AsyncWorkers = AsyncWorkers0#{WorkerPid => WorkerMRef}, + AsyncWorkerMRef = monitor(process, AsyncWorkerPid), + AsyncWorkers = AsyncWorkers0#{AsyncWorkerPid => AsyncWorkerMRef}, Data = Data0#{async_workers := AsyncWorkers}, - {Data, WorkerMRef}; + {Data, AsyncWorkerMRef}; ensure_async_worker_monitored(Data0, _Result) -> {Data0, undefined}. -spec store_async_worker_reference(undefined | ets:tid(), inflight_key(), undefined | reference()) -> ok. -store_async_worker_reference(undefined = _InflightTID, _Ref, _WorkerMRef) -> +store_async_worker_reference(undefined = _InflightTID, _Ref, _AsyncWorkerMRef) -> ok; store_async_worker_reference(_InflightTID, _Ref, undefined = _WorkerRef) -> ok; -store_async_worker_reference(InflightTID, Ref, WorkerMRef) when - is_reference(WorkerMRef) +store_async_worker_reference(InflightTID, Ref, AsyncWorkerMRef) when + is_reference(AsyncWorkerMRef) -> _ = ets:update_element( - InflightTID, Ref, {?WORKER_MREF_IDX, WorkerMRef} + InflightTID, Ref, {?WORKER_MREF_IDX, AsyncWorkerMRef} ), ok. -ack_inflight(undefined, _Ref, _WorkerPid) -> +ack_inflight(undefined, _Ref, _BufferWorkerPid) -> false; -ack_inflight(InflightTID, Ref, WorkerPid) -> +ack_inflight(InflightTID, Ref, BufferWorkerPid) -> {Count, Removed} = case ets:take(InflightTID, Ref) of - [?INFLIGHT_ITEM(Ref, ?QUERY(_, _, _, _), _IsRetriable, _WorkerMRef)] -> + [?INFLIGHT_ITEM(Ref, ?QUERY(_, _, _, _), _IsRetriable, _AsyncWorkerMRef)] -> {1, true}; - [?INFLIGHT_ITEM(Ref, [?QUERY(_, _, _, _) | _] = Batch, _IsRetriable, _WorkerMRef)] -> + [?INFLIGHT_ITEM(Ref, [?QUERY(_, _, _, _) | _] = Batch, _IsRetriable, _AsyncWorkerMRef)] -> {length(Batch), true}; [] -> {0, false} end, FlushCheck = dec_inflight_remove(InflightTID, Count, Removed), case FlushCheck of - continue -> ok; - flush -> ?MODULE:flush_worker(WorkerPid) + no_flush -> ok; + flush -> ?MODULE:flush_worker(BufferWorkerPid) end, IsKnownRef = (Count > 0), IsKnownRef. -mark_inflight_items_as_retriable(Data, WorkerMRef) -> +mark_inflight_items_as_retriable(Data, AsyncWorkerMRef) -> #{inflight_tid := InflightTID} = Data, IsRetriable = true, MatchSpec = ets:fun2ms( - fun(?INFLIGHT_ITEM(Ref, BatchOrQuery, _IsRetriable, WorkerMRef0)) when - WorkerMRef =:= WorkerMRef0 + fun(?INFLIGHT_ITEM(Ref, BatchOrQuery, _IsRetriable, AsyncWorkerMRef0)) when + AsyncWorkerMRef =:= AsyncWorkerMRef0 -> - ?INFLIGHT_ITEM(Ref, BatchOrQuery, IsRetriable, WorkerMRef0) + ?INFLIGHT_ITEM(Ref, BatchOrQuery, IsRetriable, AsyncWorkerMRef0) end ), _NumAffected = ets:select_replace(InflightTID, MatchSpec), @@ -1559,9 +1563,9 @@ inc_inflight(InflightTID, Count) -> ok. -spec dec_inflight_remove(undefined | ets:tid(), non_neg_integer(), Removed :: boolean()) -> - continue | flush. + no_flush | flush. dec_inflight_remove(_InflightTID, _Count = 0, _Removed = false) -> - continue; + no_flush; dec_inflight_remove(InflightTID, _Count = 0, _Removed = true) -> NewValue = ets:update_counter(InflightTID, ?BATCH_COUNT_REF, {2, -1, 0, 0}), MaxValue = emqx_utils_ets:lookup_value(InflightTID, ?MAX_SIZE_REF, 0), @@ -1570,7 +1574,7 @@ dec_inflight_remove(InflightTID, _Count = 0, _Removed = true) -> %% make it continue flushing. case NewValue =:= MaxValue - 1 of true -> flush; - false -> continue + false -> no_flush end; dec_inflight_remove(InflightTID, Count, _Removed = true) when Count > 0 -> %% If Count > 0, it must have been removed @@ -1582,7 +1586,7 @@ dec_inflight_remove(InflightTID, Count, _Removed = true) when Count > 0 -> %% make it continue flushing. case NewValue =:= MaxValue - 1 of true -> flush; - false -> continue + false -> no_flush end. dec_inflight_update(_InflightTID, _Count = 0) -> From 9c71f4ecbd13591fe1bec0b5f3238ac734c6db2f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 19 May 2023 09:22:54 -0300 Subject: [PATCH 139/144] test: fix flaky test --- apps/emqx_bridge_clickhouse/etc/emqx_bridge_clickhouse.conf | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 apps/emqx_bridge_clickhouse/etc/emqx_bridge_clickhouse.conf diff --git a/apps/emqx_bridge_clickhouse/etc/emqx_bridge_clickhouse.conf b/apps/emqx_bridge_clickhouse/etc/emqx_bridge_clickhouse.conf new file mode 100644 index 000000000..e69de29bb From 0559d6f6396630e3ad03afd6471ceac5dfaec7af Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 19 May 2023 16:39:03 -0300 Subject: [PATCH 140/144] refactor(buffer_worker): use static fn for bumping counters --- .../src/emqx_resource_buffer_worker.erl | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 47769418b..35761822d 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -936,27 +936,34 @@ ensure_metrics_flush_timer(Data = #{metrics_tref := undefined, metrics_flush_int -spec bump_counters(id(), counters()) -> ok. bump_counters(Id, Counters) -> - maps:foreach( - fun - (dropped_expired, Val) -> - emqx_resource_metrics:dropped_expired_inc(Id, Val); - (dropped_queue_full, Val) -> - emqx_resource_metrics:dropped_queue_full_inc(Id, Val); - (failed, Val) -> - emqx_resource_metrics:failed_inc(Id, Val); - (retried_failed, Val) -> - emqx_resource_metrics:retried_failed_inc(Id, Val); - (success, Val) -> - emqx_resource_metrics:success_inc(Id, Val); - (retried_success, Val) -> - emqx_resource_metrics:retried_success_inc(Id, Val); - (dropped_resource_not_found, Val) -> - emqx_resource_metrics:dropped_resource_not_found_inc(Id, Val); - (dropped_resource_stopped, Val) -> - emqx_resource_metrics:dropped_resource_stopped_inc(Id, Val) - end, - Counters - ). + Iter = maps:iterator(Counters), + do_bump_counters(Iter, Id). + +do_bump_counters(Iter, Id) -> + case maps:next(Iter) of + {Key, Val, NIter} -> + do_bump_counters1(Key, Val, Id), + do_bump_counters(NIter, Id); + none -> + ok + end. + +do_bump_counters1(dropped_expired, Val, Id) -> + emqx_resource_metrics:dropped_expired_inc(Id, Val); +do_bump_counters1(dropped_queue_full, Val, Id) -> + emqx_resource_metrics:dropped_queue_full_inc(Id, Val); +do_bump_counters1(failed, Val, Id) -> + emqx_resource_metrics:failed_inc(Id, Val); +do_bump_counters1(retried_failed, Val, Id) -> + emqx_resource_metrics:retried_failed_inc(Id, Val); +do_bump_counters1(success, Val, Id) -> + emqx_resource_metrics:success_inc(Id, Val); +do_bump_counters1(retried_success, Val, Id) -> + emqx_resource_metrics:retried_success_inc(Id, Val); +do_bump_counters1(dropped_resource_not_found, Val, Id) -> + emqx_resource_metrics:dropped_resource_not_found_inc(Id, Val); +do_bump_counters1(dropped_resource_stopped, Val, Id) -> + emqx_resource_metrics:dropped_resource_stopped_inc(Id, Val). -spec set_gauges(data()) -> ok. set_gauges(_Data = #{id := Id, index := Index, queue := Q, inflight_tid := InflightTID}) -> From 75c8577b6b6a54febf5545122d271ee61eae55dd Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 19 May 2023 18:56:12 +0200 Subject: [PATCH 141/144] build(./dev): export SCHEMA_MOD SCHEMA_MOD is used at runtime in EMQX Erlang code whenc loading configuration --- dev | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dev b/dev index 3ac8cb5c7..0b16f0082 100755 --- a/dev +++ b/dev @@ -131,10 +131,10 @@ export PROFILE case "${PROFILE}" in emqx) - SCHEMA_MOD='emqx_conf_schema' + export SCHEMA_MOD='emqx_conf_schema' ;; emqx-enterprise) - SCHEMA_MOD='emqx_ee_conf_schema' + export SCHEMA_MOD='emqx_ee_conf_schema' ;; esac @@ -329,11 +329,10 @@ boot() { copy_other_conf_files APPS="$(apps_to_load)" - BOOT_SEQUENCE=" Apps=[${APPS}], ok=lists:foreach(fun application:load/1, Apps), - io:format(user, \"~nLoaded ~p apps~n\", [length(Apps)]), + io:format(user, \"~nLoaded: ~p~n\", [Apps]), {ok, _} = application:ensure_all_started(emqx_machine). " From 3482c2c849d831d51ce4dca9d1b049b1de8b6f8d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 19 May 2023 19:47:22 +0200 Subject: [PATCH 142/144] refactor: redefine default value for 'node.applications' in enterprise --- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src | 2 +- .../emqx_ee_conf/src/emqx_ee_conf_schema.erl | 38 ++++++++++++++++--- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 97efa67cc..6adb456ca 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -560,7 +560,7 @@ fields("node") -> emqx_schema:comma_separated_atoms(), #{ mapping => "emqx_machine.applications", - default => [], + default => <<"">>, 'readOnly' => true, importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(node_applications) diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src index 3df18ce7a..599b0798c 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_conf, [ {description, "EMQX Enterprise Edition configuration schema"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index c1b1a002c..c407836fb 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -14,14 +14,18 @@ namespace() -> emqx_conf_schema:namespace(). roots() -> - lists:foldl( - fun(Module, Roots) -> - Roots ++ apply(Module, roots, []) - end, - emqx_conf_schema:roots(), - ?EE_SCHEMA_MODULES + redefine_roots( + lists:foldl( + fun(Module, Roots) -> + Roots ++ apply(Module, roots, []) + end, + emqx_conf_schema:roots(), + ?EE_SCHEMA_MODULES + ) ). +fields("node") -> + redefine_node(emqx_conf_schema:fields("node")); fields(Name) -> emqx_conf_schema:fields(Name). @@ -33,3 +37,25 @@ translation(Name) -> validations() -> emqx_conf_schema:validations(). + +redefine_node(Fields) -> + Overrides = [{"applications", #{default => <<"emqx_license">>}}], + override(Fields, Overrides). + +redefine_roots(Roots) -> + Overrides = [{"node", #{type => hoconsc:ref(?MODULE, "node")}}], + override(Roots, Overrides). + +override(Fields, []) -> + Fields; +override(Fields, [{Name, Override}]) -> + Schema = find_schema(Name, Fields), + NewSchema = hocon_schema:override(Schema, Override), + replace_schema(Name, NewSchema, Fields). + +find_schema(Name, Fields) -> + {Name, Schema} = lists:keyfind(Name, 1, Fields), + Schema. + +replace_schema(Name, Schema, Fields) -> + lists:keyreplace(Name, 1, Fields, {Name, Schema}). From 40e8d5d039ae19060bc5766cb248d061b6c488fe Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 19 May 2023 20:01:30 +0200 Subject: [PATCH 143/144] refactor: rename lib-ee/emqx_ee_conf to apps/emqx_enterprise --- apps/emqx/test/emqx_common_test_helpers.erl | 4 +- .../emqx_bridge_kafka_impl_consumer_SUITE.erl | 2 +- ...emqx_bridge_pulsar_impl_producer_SUITE.erl | 2 +- .../emqx_enterprise}/.gitignore | 0 apps/emqx_enterprise/README.md | 6 +++ .../emqx_enterprise}/etc/emqx-enterprise.conf | 0 .../emqx_enterprise}/rebar.config | 0 .../src/emqx_enterprise.app.src | 6 +-- .../src/emqx_enterprise_schema.erl | 12 +++-- .../test/emqx_enterprise_schema_SUITE.erl | 52 ++++++++++++++++++ .../test/emqx_enterprise_schema_tests.erl | 4 +- build | 2 +- dev | 2 +- lib-ee/emqx_ee_conf/README.md | 3 -- .../test/emqx_ee_conf_schema_SUITE.erl | 53 ------------------- .../test/emqx_ee_schema_registry_SUITE.erl | 2 +- .../emqx_license/test/emqx_license_SUITE.erl | 6 +-- mix.exs | 4 +- rebar.config.erl | 4 +- 19 files changed, 85 insertions(+), 79 deletions(-) rename {lib-ee/emqx_ee_conf => apps/emqx_enterprise}/.gitignore (100%) create mode 100644 apps/emqx_enterprise/README.md rename {lib-ee/emqx_ee_conf => apps/emqx_enterprise}/etc/emqx-enterprise.conf (100%) rename {lib-ee/emqx_ee_conf => apps/emqx_enterprise}/rebar.config (100%) rename lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src => apps/emqx_enterprise/src/emqx_enterprise.app.src (53%) rename lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl => apps/emqx_enterprise/src/emqx_enterprise_schema.erl (86%) create mode 100644 apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl rename lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl => apps/emqx_enterprise/test/emqx_enterprise_schema_tests.erl (85%) delete mode 100644 lib-ee/emqx_ee_conf/README.md delete mode 100644 lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index d8e0690f2..c22944209 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -859,8 +859,8 @@ setup_node(Node, Opts) when is_map(Opts) -> LoadSchema andalso begin %% to avoid sharing data between executions and/or - %% nodes. these variables might notbe in the - %% config file (e.g.: emqx_ee_conf_schema). + %% nodes. these variables might not be in the + %% config file (e.g.: emqx_enterprise_schema). NodeDataDir = filename:join([ PrivDataDir, node(), diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 4f98f33cf..c17d21635 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -1076,7 +1076,7 @@ cluster(Config) -> {priv_data_dir, PrivDataDir}, {load_schema, true}, {start_autocluster, true}, - {schema_mod, emqx_ee_conf_schema}, + {schema_mod, emqx_enterprise_schema}, {env_handler, fun (emqx) -> application:set_env(emqx, boot_modules, [broker, router]), diff --git a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl index be38f6625..76d9f94e1 100644 --- a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl @@ -496,7 +496,7 @@ cluster(Config) -> {priv_data_dir, PrivDataDir}, {load_schema, true}, {start_autocluster, true}, - {schema_mod, emqx_ee_conf_schema}, + {schema_mod, emqx_enterprise_schema}, {env_handler, fun (emqx) -> application:set_env(emqx, boot_modules, [broker, router]), diff --git a/lib-ee/emqx_ee_conf/.gitignore b/apps/emqx_enterprise/.gitignore similarity index 100% rename from lib-ee/emqx_ee_conf/.gitignore rename to apps/emqx_enterprise/.gitignore diff --git a/apps/emqx_enterprise/README.md b/apps/emqx_enterprise/README.md new file mode 100644 index 000000000..06f249581 --- /dev/null +++ b/apps/emqx_enterprise/README.md @@ -0,0 +1,6 @@ +# EMQX Enterprise Application + +This application so fart only holds EMQX config schema for enterprise edition. +In the future this application will collect more responsibilities in managing +enterprise edition specific features. + diff --git a/lib-ee/emqx_ee_conf/etc/emqx-enterprise.conf b/apps/emqx_enterprise/etc/emqx-enterprise.conf similarity index 100% rename from lib-ee/emqx_ee_conf/etc/emqx-enterprise.conf rename to apps/emqx_enterprise/etc/emqx-enterprise.conf diff --git a/lib-ee/emqx_ee_conf/rebar.config b/apps/emqx_enterprise/rebar.config similarity index 100% rename from lib-ee/emqx_ee_conf/rebar.config rename to apps/emqx_enterprise/rebar.config diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/apps/emqx_enterprise/src/emqx_enterprise.app.src similarity index 53% rename from lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src rename to apps/emqx_enterprise/src/emqx_enterprise.app.src index 599b0798c..84c3ffc02 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src +++ b/apps/emqx_enterprise/src/emqx_enterprise.app.src @@ -1,6 +1,6 @@ -{application, emqx_ee_conf, [ - {description, "EMQX Enterprise Edition configuration schema"}, - {vsn, "0.1.3"}, +{application, emqx_enterprise, [ + {description, "EMQX Enterprise Edition"}, + {vsn, "0.1.0"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl similarity index 86% rename from lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl rename to apps/emqx_enterprise/src/emqx_enterprise_schema.erl index c407836fb..971d1a56f 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl @@ -2,11 +2,11 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_conf_schema). +-module(emqx_enterprise_schema). -behaviour(hocon_schema). --export([namespace/0, roots/0, fields/1, translations/0, translation/1, validations/0]). +-export([namespace/0, roots/0, fields/1, translations/0, translation/1, validations/0, desc/1]). -define(EE_SCHEMA_MODULES, [emqx_license_schema, emqx_ee_schema_registry_schema]). @@ -48,10 +48,11 @@ redefine_roots(Roots) -> override(Fields, []) -> Fields; -override(Fields, [{Name, Override}]) -> +override(Fields, [{Name, Override} | More]) -> Schema = find_schema(Name, Fields), NewSchema = hocon_schema:override(Schema, Override), - replace_schema(Name, NewSchema, Fields). + NewFields = replace_schema(Name, NewSchema, Fields), + override(NewFields, More). find_schema(Name, Fields) -> {Name, Schema} = lists:keyfind(Name, 1, Fields), @@ -59,3 +60,6 @@ find_schema(Name, Fields) -> replace_schema(Name, Schema, Fields) -> lists:keyreplace(Name, 1, Fields, {Name, Schema}). + +desc(Name) -> + emqx_conf_schema:desc(Name). diff --git a/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl b/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl new file mode 100644 index 000000000..e7f4c0075 --- /dev/null +++ b/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl @@ -0,0 +1,52 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_enterprise_schema_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +%%------------------------------------------------------------------------------ +%% Tests +%%------------------------------------------------------------------------------ + +t_namespace(_Config) -> + ?assertEqual( + emqx_conf_schema:namespace(), + emqx_enterprise_schema:namespace() + ). + +t_roots(_Config) -> + EnterpriseRoots = emqx_enterprise_schema:roots(), + ?assertMatch({license, _}, lists:keyfind(license, 1, EnterpriseRoots)). + +t_fields(_Config) -> + CeFields = emqx_conf_schema:fields("node"), + EeFields = emqx_enterprise_schema:fields("node"), + ?assertEqual(length(CeFields), length(EeFields)), + lists:foreach( + fun({{CeName, CeSchema}, {EeName, EeSchema}}) -> + ?assertEqual(CeName, EeName), + case CeName of + "applications" -> + ok; + _ -> + ?assertEqual({CeName, CeSchema}, {EeName, EeSchema}) + end + end, + lists:zip(CeFields, EeFields) + ). + +t_translations(_Config) -> + [Root | _] = emqx_enterprise_schema:translations(), + ?assertEqual( + emqx_conf_schema:translation(Root), + emqx_enterprise_schema:translation(Root) + ). diff --git a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl b/apps/emqx_enterprise/test/emqx_enterprise_schema_tests.erl similarity index 85% rename from lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl rename to apps/emqx_enterprise/test/emqx_enterprise_schema_tests.erl index 5e1d4e551..a78bbcb2e 100644 --- a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl +++ b/apps/emqx_enterprise/test/emqx_enterprise_schema_tests.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_conf_schema_tests). +-module(emqx_enterprise_schema_tests). -include_lib("eunit/include/eunit.hrl"). @@ -22,7 +22,7 @@ doc_gen_test() -> "priv", "i18n.conf" ]), - _ = emqx_conf:dump_schema(Dir, emqx_ee_conf_schema, I18nFile), + _ = emqx_conf:dump_schema(Dir, emqx_enterprise_schema, I18nFile), ok end }. diff --git a/build b/build index 0846d6057..5e396fdd2 100755 --- a/build +++ b/build @@ -106,7 +106,7 @@ make_docs() { fi case "$(is_enterprise "$PROFILE")" in 'yes') - SCHEMA_MODULE='emqx_ee_conf_schema' + SCHEMA_MODULE='emqx_enterprise_schema' ;; 'no') SCHEMA_MODULE='emqx_conf_schema' diff --git a/dev b/dev index 0b16f0082..425da993a 100755 --- a/dev +++ b/dev @@ -134,7 +134,7 @@ case "${PROFILE}" in export SCHEMA_MOD='emqx_conf_schema' ;; emqx-enterprise) - export SCHEMA_MOD='emqx_ee_conf_schema' + export SCHEMA_MOD='emqx_enterprise_schema' ;; esac diff --git a/lib-ee/emqx_ee_conf/README.md b/lib-ee/emqx_ee_conf/README.md deleted file mode 100644 index 701d285cc..000000000 --- a/lib-ee/emqx_ee_conf/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# emqx_ee_conf - -EMQX Enterprise configuration schema diff --git a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl deleted file mode 100644 index 941c3e4d1..000000000 --- a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl +++ /dev/null @@ -1,53 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- - --module(emqx_ee_conf_schema_SUITE). - --compile(nowarn_export_all). --compile(export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - -all() -> - emqx_common_test_helpers:all(?MODULE). - -%%------------------------------------------------------------------------------ -%% Tests -%%------------------------------------------------------------------------------ - -t_namespace(_Config) -> - ?assertEqual( - emqx_conf_schema:namespace(), - emqx_ee_conf_schema:namespace() - ). - -t_roots(_Config) -> - BaseRoots = emqx_conf_schema:roots(), - EnterpriseRoots = emqx_ee_conf_schema:roots(), - - ?assertEqual([], BaseRoots -- EnterpriseRoots), - - ?assert( - lists:any( - fun - ({license, _}) -> true; - (_) -> false - end, - EnterpriseRoots - ) - ). - -t_fields(_Config) -> - ?assertEqual( - emqx_conf_schema:fields("node"), - emqx_ee_conf_schema:fields("node") - ). - -t_translations(_Config) -> - [Root | _] = emqx_ee_conf_schema:translations(), - ?assertEqual( - emqx_conf_schema:translation(Root), - emqx_ee_conf_schema:translation(Root) - ). diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl index 7ad01fa06..d17c159c3 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -345,7 +345,7 @@ cluster(Config) -> {priv_data_dir, PrivDataDir}, {load_schema, true}, {start_autocluster, true}, - {schema_mod, emqx_ee_conf_schema}, + {schema_mod, emqx_enterprise_schema}, %% need to restart schema registry app in the tests so %% that it re-registers the config handler that is lost %% when emqx_conf restarts during join. diff --git a/lib-ee/emqx_license/test/emqx_license_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_SUITE.erl index 4a0f6d91b..69adabe76 100644 --- a/lib-ee/emqx_license/test/emqx_license_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_SUITE.erl @@ -73,10 +73,10 @@ setup_test(TestCase, Config) when [ {apps, [emqx_conf, emqx_license]}, {load_schema, false}, - {schema_mod, emqx_ee_conf_schema}, + {schema_mod, emqx_enterprise_schema}, {env_handler, fun (emqx) -> - emqx_config:save_schema_mod_and_names(emqx_ee_conf_schema), + emqx_config:save_schema_mod_and_names(emqx_enterprise_schema), %% emqx_config:save_schema_mod_and_names(emqx_license_schema), application:set_env(emqx, boot_modules, []), application:set_env( @@ -90,7 +90,7 @@ setup_test(TestCase, Config) when ), ok; (emqx_conf) -> - emqx_config:save_schema_mod_and_names(emqx_ee_conf_schema), + emqx_config:save_schema_mod_and_names(emqx_enterprise_schema), %% emqx_config:save_schema_mod_and_names(emqx_license_schema), application:set_env( emqx, diff --git a/mix.exs b/mix.exs index 717172041..c7fc1e3c0 100644 --- a/mix.exs +++ b/mix.exs @@ -384,7 +384,7 @@ defmodule EMQXUmbrella.MixProject do if(edition_type == :enterprise, do: [ emqx_license: :permanent, - emqx_ee_conf: :load, + emqx_enterprise: :load, emqx_ee_connector: :permanent, emqx_ee_bridge: :permanent, emqx_bridge_kafka: :permanent, @@ -785,7 +785,7 @@ defmodule EMQXUmbrella.MixProject do end end - defp emqx_schema_mod(:enterprise), do: :emqx_ee_conf_schema + defp emqx_schema_mod(:enterprise), do: :emqx_enterprise_schema defp emqx_schema_mod(:community), do: :emqx_conf_schema defp bcrypt_dep() do diff --git a/rebar.config.erl b/rebar.config.erl index d556b41aa..a740ab183 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -346,7 +346,7 @@ overlay_vars_edition(ce) -> ]; overlay_vars_edition(ee) -> [ - {emqx_schema_mod, emqx_ee_conf_schema}, + {emqx_schema_mod, emqx_enterprise_schema}, {is_enterprise, "yes"} ]. @@ -453,7 +453,7 @@ is_app(Name) -> relx_apps_per_edition(ee) -> [ emqx_license, - {emqx_ee_conf, load}, + {emqx_enterprise, load}, emqx_ee_connector, emqx_ee_bridge, emqx_bridge_kafka, From 0b9a32f49adaf4135dd733c40985befc02c547ec Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 19 May 2023 20:13:01 +0200 Subject: [PATCH 144/144] chore: add BSL in apps/emqx_enterprise --- apps/emqx_enterprise/BSL.txt | 94 ++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 apps/emqx_enterprise/BSL.txt diff --git a/apps/emqx_enterprise/BSL.txt b/apps/emqx_enterprise/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_enterprise/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way.