From f9a75cb2a9f5404803af26ae02376becfe9c3578 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Tue, 16 May 2023 20:48:54 -0300 Subject: [PATCH 01/20] fix(oracle): hide password on stacktrace --- apps/emqx_oracle/rebar.config | 2 +- apps/emqx_oracle/src/emqx_oracle.app.src | 2 +- apps/emqx_oracle/src/emqx_oracle.erl | 6 ++---- changes/ee/fix-10741.en.md | 1 + 4 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 changes/ee/fix-10741.en.md diff --git a/apps/emqx_oracle/rebar.config b/apps/emqx_oracle/rebar.config index 14461ba34..c6113fe30 100644 --- a/apps/emqx_oracle/rebar.config +++ b/apps/emqx_oracle/rebar.config @@ -1,7 +1,7 @@ %% -*- mode: erlang; -*- {erl_opts, [debug_info]}. -{deps, [ {jamdb_oracle, {git, "https://github.com/emqx/jamdb_oracle", {tag, "0.4.9.4"}}} +{deps, [ {jamdb_oracle, {git, "https://github.com/emqx/jamdb_oracle", {tag, "0.4.9.5"}}} , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} ]}. diff --git a/apps/emqx_oracle/src/emqx_oracle.app.src b/apps/emqx_oracle/src/emqx_oracle.app.src index fa48e8479..3beda05a4 100644 --- a/apps/emqx_oracle/src/emqx_oracle.app.src +++ b/apps/emqx_oracle/src/emqx_oracle.app.src @@ -1,6 +1,6 @@ {application, emqx_oracle, [ {description, "EMQX Enterprise Oracle Database Connector"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl index a0d7169f3..70d9cfbb2 100644 --- a/apps/emqx_oracle/src/emqx_oracle.erl +++ b/apps/emqx_oracle/src/emqx_oracle.erl @@ -96,7 +96,7 @@ on_start( {host, Host}, {port, Port}, {user, emqx_plugin_libs_rule:str(User)}, - {password, emqx_secret:wrap(maps:get(password, Config, ""))}, + {password, jamdb_secret:wrap(maps:get(password, Config, ""))}, {sid, emqx_plugin_libs_rule:str(Sid)}, {service_name, emqx_plugin_libs_rule:str(ServiceName)}, {database, DB}, @@ -258,9 +258,7 @@ oracle_host_options() -> ?ORACLE_HOST_OPTIONS. connect(Opts) -> - Password = emqx_secret:unwrap(proplists:get_value(password, Opts)), - NewOpts = lists:keyreplace(password, 1, Opts, {password, Password}), - jamdb_oracle:start_link(NewOpts). + jamdb_oracle:start_link(Opts). sql_query_to_str(SqlQuery) -> emqx_plugin_libs_rule:str(SqlQuery). diff --git a/changes/ee/fix-10741.en.md b/changes/ee/fix-10741.en.md new file mode 100644 index 000000000..d6d1e27ac --- /dev/null +++ b/changes/ee/fix-10741.en.md @@ -0,0 +1 @@ +Fix password leaking on stacktrace for Oracle Database. From 7cad1cb93a7614661a5c529e950664a073cb3c3b Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Wed, 24 May 2023 18:18:32 +0800 Subject: [PATCH 02/20] fix: set the base_url in IoTDB bridge as a required field --- apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl index aa2c32589..1a82e0fb6 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl @@ -130,6 +130,7 @@ request_config() -> mk( emqx_schema:url(), #{ + required => true, desc => ?DESC("config_base_url") } )}, From 4f21bdb885ecf181e25ed8b1bdf9e30a030e1fc1 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 May 2023 13:05:46 +0200 Subject: [PATCH 03/20] fix: REST API version 2 not available in IoTDB 0.13 and 1.0 This commit makes sure that REST API version 1 is used when the user has selected to use IoTDB 0.13 or 1.0 Fixes: https://emqx.atlassian.net/browse/EMQX-9920 --- apps/emqx_bridge/src/emqx_bridge_resource.erl | 23 ++++++++++++++----- .../include/emqx_bridge_iotdb.hrl | 3 ++- .../src/emqx_bridge_iotdb.erl | 6 ++--- .../src/emqx_bridge_iotdb_impl.erl | 14 +++++++---- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 60ee242d1..e622403b3 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -57,11 +57,6 @@ (TYPE) =:= <<"kafka_consumer">> orelse ?IS_BI_DIR_BRIDGE(TYPE) ). -%% [FIXME] this has no place here, it's used in parse_confs/3, which should -%% rather delegate to a behavior callback than implementing domain knowledge -%% here (reversed dependency) --define(INSERT_TABLET_PATH, "/rest/v2/insertTablet"). - -if(?EMQX_RELEASE_EDITION == ee). bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt; bridge_to_resource_type(mqtt) -> emqx_connector_mqtt; @@ -343,6 +338,11 @@ parse_confs( } }; parse_confs(<<"iotdb">>, Name, Conf) -> + %% [FIXME] this has no place here, it's used in parse_confs/3, which should + %% rather delegate to a behavior callback than implementing domain knowledge + %% here (reversed dependency) + InsertTabletPathV1 = <<"rest/v1/insertTablet">>, + InsertTabletPathV2 = <<"rest/v2/insertTablet">>, #{ base_url := BaseURL, authentication := @@ -352,10 +352,21 @@ parse_confs(<<"iotdb">>, Name, Conf) -> } } = Conf, BasicToken = base64:encode(<>), + %% This version atom correspond to the macro ?VSN_1_1_X in + %% emqx_bridge_iotdb.hrl. It would be better to use the macro directly, but + %% this cannot be done without introducing a dependency on the + %% emqx_iotdb_bridge app (which is an EE app). + DefaultIOTDBBridge = 'v1.1.x', + Version = maps:get(iotdb_version, Conf, DefaultIOTDBBridge), + InsertTabletPath = + case Version of + DefaultIOTDBBridge -> InsertTabletPathV2; + _ -> InsertTabletPathV1 + end, WebhookConfig = Conf#{ method => <<"post">>, - url => <>, + url => <>, headers => [ {<<"Content-type">>, <<"application/json">>}, {<<"Authorization">>, BasicToken} diff --git a/apps/emqx_bridge_iotdb/include/emqx_bridge_iotdb.hrl b/apps/emqx_bridge_iotdb/include/emqx_bridge_iotdb.hrl index 5d693547a..5e5048202 100644 --- a/apps/emqx_bridge_iotdb/include/emqx_bridge_iotdb.hrl +++ b/apps/emqx_bridge_iotdb/include/emqx_bridge_iotdb.hrl @@ -5,7 +5,8 @@ -ifndef(EMQX_BRIDGE_IOTDB_HRL). -define(EMQX_BRIDGE_IOTDB_HRL, true). --define(VSN_1_X, 'v1.x'). +-define(VSN_1_1_X, 'v1.1.x'). +-define(VSN_1_0_X, 'v1.0.x'). -define(VSN_0_13_X, 'v0.13.x'). -endif. diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl index aa2c32589..68bbc7c42 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl @@ -109,10 +109,10 @@ basic_config() -> )}, {iotdb_version, mk( - hoconsc:enum([?VSN_1_X, ?VSN_0_13_X]), + hoconsc:enum([?VSN_1_1_X, ?VSN_1_0_X, ?VSN_0_13_X]), #{ desc => ?DESC("config_iotdb_version"), - default => ?VSN_1_X + default => ?VSN_1_1_X } )} ] ++ resource_creation_opts() ++ @@ -217,7 +217,7 @@ conn_bridge_example(_Method, Type) -> is_aligned => false, device_id => <<"my_device">>, base_url => <<"http://iotdb.local:18080/">>, - iotdb_version => ?VSN_1_X, + iotdb_version => ?VSN_1_1_X, connect_timeout => <<"15s">>, pool_type => <<"random">>, pool_size => 8, diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_impl.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_impl.erl index 8331e715f..f19fc0839 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_impl.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_impl.erl @@ -280,7 +280,7 @@ make_iotdb_insert_request(MessageUnparsedPayload, State) -> Message = maps:update_with(payload, fun make_parsed_payload/1, MessageUnparsedPayload), IsAligned = maps:get(is_aligned, State, false), DeviceId = device_id(Message, State), - IotDBVsn = maps:get(iotdb_version, State, ?VSN_1_X), + IotDBVsn = maps:get(iotdb_version, State, ?VSN_1_1_X), Payload = make_list(maps:get(payload, Message)), PreProcessedData = preproc_data_list(Payload), DataList = proc_data(PreProcessedData, Message), @@ -349,15 +349,21 @@ insert_value(1, Data, [Value | Values]) -> insert_value(Index, Data, [Value | Values]) -> [[null | Value] | insert_value(Index - 1, Data, Values)]. -iotdb_field_key(is_aligned, ?VSN_1_X) -> +iotdb_field_key(is_aligned, ?VSN_1_1_X) -> + <<"is_aligned">>; +iotdb_field_key(is_aligned, ?VSN_1_0_X) -> <<"is_aligned">>; iotdb_field_key(is_aligned, ?VSN_0_13_X) -> <<"isAligned">>; -iotdb_field_key(device_id, ?VSN_1_X) -> +iotdb_field_key(device_id, ?VSN_1_1_X) -> + <<"device">>; +iotdb_field_key(device_id, ?VSN_1_0_X) -> <<"device">>; iotdb_field_key(device_id, ?VSN_0_13_X) -> <<"deviceId">>; -iotdb_field_key(data_types, ?VSN_1_X) -> +iotdb_field_key(data_types, ?VSN_1_1_X) -> + <<"data_types">>; +iotdb_field_key(data_types, ?VSN_1_0_X) -> <<"data_types">>; iotdb_field_key(data_types, ?VSN_0_13_X) -> <<"dataTypes">>. From 7374e00a02432c8f79e2cc20bb940c71a3049c6b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 May 2023 13:13:54 -0300 Subject: [PATCH 04/20] test(kafka_consumer): attempt to stabilize cluster tests Example failure: https://github.com/emqx/emqx/actions/runs/5070096710/jobs/9105822319#step:7:515 The attempt here is to setup the spy as early as possible, before the bridge starts, so we avoid missing rebalancing events. --- .../emqx_bridge_kafka_impl_consumer_SUITE.erl | 61 +++++++++---------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 4f98f33cf..42729c002 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -994,36 +994,33 @@ reconstruct_assignments_from_events(KafkaTopic, Events0, Acc0) -> Assignments ). -setup_group_subscriber_spy(Node) -> +setup_group_subscriber_spy_fn() -> TestPid = self(), - ok = erpc:call( - Node, - fun() -> - ok = meck:new(brod_group_subscriber_v2, [ - passthrough, no_link, no_history, non_strict - ]), - ok = meck:expect( - brod_group_subscriber_v2, - assignments_received, - fun(Pid, MemberId, GenerationId, TopicAssignments) -> - ?tp( - kafka_assignment, - #{ - node => node(), - pid => Pid, - member_id => MemberId, - generation_id => GenerationId, - topic_assignments => TopicAssignments - } - ), - TestPid ! - {kafka_assignment, node(), {Pid, MemberId, GenerationId, TopicAssignments}}, - meck:passthrough([Pid, MemberId, GenerationId, TopicAssignments]) - end - ), - ok - end - ). + fun() -> + ok = meck:new(brod_group_subscriber_v2, [ + passthrough, no_link, no_history, non_strict + ]), + ok = meck:expect( + brod_group_subscriber_v2, + assignments_received, + fun(Pid, MemberId, GenerationId, TopicAssignments) -> + ?tp( + kafka_assignment, + #{ + node => node(), + pid => Pid, + member_id => MemberId, + generation_id => GenerationId, + topic_assignments => TopicAssignments + } + ), + TestPid ! + {kafka_assignment, node(), {Pid, MemberId, GenerationId, TopicAssignments}}, + meck:passthrough([Pid, MemberId, GenerationId, TopicAssignments]) + end + ), + ok + end. wait_for_cluster_rpc(Node) -> %% need to wait until the config handler is ready after @@ -1067,6 +1064,7 @@ cluster(Config) -> _ -> ct_slave end, + ExtraEnvHandlerHook = setup_group_subscriber_spy_fn(), Cluster = emqx_common_test_helpers:emqx_cluster( [core, core], [ @@ -1080,6 +1078,7 @@ cluster(Config) -> {env_handler, fun (emqx) -> application:set_env(emqx, boot_modules, [broker, router]), + ExtraEnvHandlerHook(), ok; (emqx_conf) -> ok; @@ -1680,7 +1679,6 @@ t_cluster_group(Config) -> Nodes ) end), - lists:foreach(fun setup_group_subscriber_spy/1, Nodes), {ok, SRef0} = snabbkaffe:subscribe( ?match_event(#{?snk_kind := kafka_consumer_subscriber_started}), length(Nodes), @@ -1757,7 +1755,6 @@ t_node_joins_existing_cluster(Config) -> ct:pal("stopping ~p", [N1]), ok = emqx_common_test_helpers:stop_slave(N1) end), - setup_group_subscriber_spy(N1), {{ok, _}, {ok, _}} = ?wait_async_action( erpc:call(N1, fun() -> @@ -1801,7 +1798,6 @@ t_node_joins_existing_cluster(Config) -> ct:pal("stopping ~p", [N2]), ok = emqx_common_test_helpers:stop_slave(N2) end), - setup_group_subscriber_spy(N2), Nodes = [N1, N2], wait_for_cluster_rpc(N2), @@ -1902,7 +1898,6 @@ t_cluster_node_down(Config) -> Nodes ) end), - lists:foreach(fun setup_group_subscriber_spy/1, Nodes), {ok, SRef0} = snabbkaffe:subscribe( ?match_event(#{?snk_kind := kafka_consumer_subscriber_started}), length(Nodes), From 8d3d3e523b523e606da62e8aa3be1e7734e85879 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Tue, 23 May 2023 12:16:23 -0300 Subject: [PATCH 05/20] fix(oracle): replace 'database' field by 'service name' Fixes https://emqx.atlassian.net/browse/EMQX-9980 --- .../src/emqx_bridge_oracle.app.src | 2 +- .../emqx_bridge_oracle/src/emqx_bridge_oracle.erl | 4 ++-- .../test/emqx_bridge_oracle_SUITE.erl | 6 ++---- apps/emqx_oracle/src/emqx_oracle.erl | 12 +++++++----- apps/emqx_oracle/src/emqx_oracle_schema.erl | 15 +++++++++++---- rel/i18n/emqx_bridge_oracle.hocon | 5 +++++ rel/i18n/emqx_oracle.hocon | 5 +++++ rel/i18n/zh/emqx_bridge_oracle.hocon | 5 +++++ rel/i18n/zh/emqx_oracle.hocon | 5 +++++ 9 files changed, 43 insertions(+), 16 deletions(-) diff --git a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src index 4f81c2110..4a2549f7c 100644 --- a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src +++ b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_oracle, [ {description, "EMQX Enterprise Oracle Database Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.erl b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.erl index 8a87f02ba..7f384c5e6 100644 --- a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.erl +++ b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.erl @@ -20,7 +20,7 @@ ]). -define(DEFAULT_SQL, << - "insert into t_mqtt_msg(msgid, topic, qos, payload)" + "insert into t_mqtt_msgs(msgid, topic, qos, payload) " "values (${id}, ${topic}, ${qos}, ${payload})" >>). @@ -41,7 +41,7 @@ values(_Method) -> name => <<"foo">>, server => <<"127.0.0.1:1521">>, pool_size => 8, - database => <<"ORCL">>, + service_name => <<"ORCL">>, sid => <<"ORCL">>, username => <<"root">>, password => <<"******">>, diff --git a/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl b/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl index b50788277..f280c51d6 100644 --- a/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl +++ b/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl @@ -14,7 +14,7 @@ -define(BRIDGE_TYPE_BIN, <<"oracle">>). -define(APPS, [emqx_bridge, emqx_resource, emqx_rule_engine, emqx_oracle, emqx_bridge_oracle]). --define(DATABASE, "XE"). +-define(SID, "XE"). -define(RULE_TOPIC, "mqtt/rule"). % -define(RULE_TOPIC_BIN, <>). @@ -196,7 +196,6 @@ oracle_config(TestCase, _ConnectionType, Config) -> io_lib:format( "bridges.oracle.~s {\n" " enable = true\n" - " database = \"~s\"\n" " sid = \"~s\"\n" " server = \"~s\"\n" " username = \"system\"\n" @@ -215,8 +214,7 @@ oracle_config(TestCase, _ConnectionType, Config) -> "}\n", [ Name, - ?DATABASE, - ?DATABASE, + ?SID, ServerURL, sql_insert_template_for_bridge() ] diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl index 70d9cfbb2..c5d8ecc77 100644 --- a/apps/emqx_oracle/src/emqx_oracle.erl +++ b/apps/emqx_oracle/src/emqx_oracle.erl @@ -75,8 +75,6 @@ on_start( InstId, #{ server := Server, - database := DB, - sid := Sid, username := User } = Config ) -> @@ -91,15 +89,19 @@ on_start( jamdb_oracle_conn:set_max_cursors_number(?MAX_CURSORS), #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, oracle_host_options()), - ServiceName = maps:get(<<"service_name">>, Config, Sid), + Sid = maps:get(sid, Config, ""), + ServiceName = + case maps:get(service_name, Config, undefined) of + undefined -> undefined; + ServiceName0 -> emqx_plugin_libs_rule:str(ServiceName0) + end, Options = [ {host, Host}, {port, Port}, {user, emqx_plugin_libs_rule:str(User)}, {password, jamdb_secret:wrap(maps:get(password, Config, ""))}, {sid, emqx_plugin_libs_rule:str(Sid)}, - {service_name, emqx_plugin_libs_rule:str(ServiceName)}, - {database, DB}, + {service_name, ServiceName}, {pool_size, maps:get(<<"pool_size">>, Config, ?DEFAULT_POOL_SIZE)}, {timeout, ?OPT_TIMEOUT}, {app_name, "EMQX Data To Oracle Database Action"} diff --git a/apps/emqx_oracle/src/emqx_oracle_schema.erl b/apps/emqx_oracle/src/emqx_oracle_schema.erl index cfa74054a..3adde5af3 100644 --- a/apps/emqx_oracle/src/emqx_oracle_schema.erl +++ b/apps/emqx_oracle/src/emqx_oracle_schema.erl @@ -19,9 +19,11 @@ roots() -> [{config, #{type => hoconsc:ref(?REF_MODULE, config)}}]. fields(config) -> - [{server, server()}, {sid, fun sid/1}] ++ - emqx_connector_schema_lib:relational_db_fields() ++ - emqx_connector_schema_lib:prepare_statement_fields(). + Fields = + [{server, server()}, {sid, fun sid/1}, {service_name, fun service_name/1}] ++ + emqx_connector_schema_lib:relational_db_fields() ++ + emqx_connector_schema_lib:prepare_statement_fields(), + proplists:delete(database, Fields). server() -> Meta = #{desc => ?DESC(?REF_MODULE, "server")}, @@ -29,5 +31,10 @@ server() -> sid(type) -> binary(); sid(desc) -> ?DESC(?REF_MODULE, "sid"); -sid(required) -> true; +sid(required) -> false; sid(_) -> undefined. + +service_name(type) -> binary(); +service_name(desc) -> ?DESC(?REF_MODULE, "service_name"); +service_name(required) -> false; +service_name(_) -> undefined. diff --git a/rel/i18n/emqx_bridge_oracle.hocon b/rel/i18n/emqx_bridge_oracle.hocon index 95e0cf4af..c0c8142e6 100644 --- a/rel/i18n/emqx_bridge_oracle.hocon +++ b/rel/i18n/emqx_bridge_oracle.hocon @@ -29,6 +29,11 @@ emqx_bridge_oracle { label = "Oracle Database Sid." } + service_name { + desc = "Service Name for Oracle Database." + label = "Oracle Database Service Name" + } + config_enable { desc = "Enable or disable this bridge" label = "Enable Or Disable Bridge" diff --git a/rel/i18n/emqx_oracle.hocon b/rel/i18n/emqx_oracle.hocon index 58de8e4c7..66224c5ec 100644 --- a/rel/i18n/emqx_oracle.hocon +++ b/rel/i18n/emqx_oracle.hocon @@ -12,4 +12,9 @@ emqx_oracle { label = "Oracle Database Sid" } + service_name { + desc = "Service Name for Oracle Database." + label = "Oracle Database Service Name" + } + } diff --git a/rel/i18n/zh/emqx_bridge_oracle.hocon b/rel/i18n/zh/emqx_bridge_oracle.hocon index 290ac6d07..5f69c6171 100644 --- a/rel/i18n/zh/emqx_bridge_oracle.hocon +++ b/rel/i18n/zh/emqx_bridge_oracle.hocon @@ -28,6 +28,11 @@ emqx_bridge_oracle { label = "Oracle Database Sid" } + service_name { + desc = "Oracle Database 服务名称。" + label = "Oracle Database 服务名称" + } + config_enable { desc = "启用/禁用桥接" label = "启用/禁用桥接" diff --git a/rel/i18n/zh/emqx_oracle.hocon b/rel/i18n/zh/emqx_oracle.hocon index 70c597cb1..06840f201 100644 --- a/rel/i18n/zh/emqx_oracle.hocon +++ b/rel/i18n/zh/emqx_oracle.hocon @@ -12,4 +12,9 @@ emqx_oracle { label = "Oracle Database Sid" } + service_name { + desc = "Oracle Database 服务名称。" + label = "Oracle Database 服务名称" + } + } From 122ebcac248f94bc44de2abc60869628b269504c Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Mon, 22 May 2023 23:01:40 -0300 Subject: [PATCH 06/20] fix: add user-friendly message when interval is out of range --- .../src/schema/emqx_resource_schema.erl | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 9b1ae64fa..48c6a3012 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -96,12 +96,14 @@ health_check_interval_range(HealthCheckInterval) when HealthCheckInterval =< ?HEALTH_CHECK_INTERVAL_RANGE_MAX -> ok; -health_check_interval_range(_HealthCheckInterval) -> - {error, #{ - msg => <<"Health Check Interval out of range">>, - min => ?HEALTH_CHECK_INTERVAL_RANGE_MIN, - max => ?HEALTH_CHECK_INTERVAL_RANGE_MAX - }}. +health_check_interval_range(HealthCheckInterval) -> + Message = get_out_of_range_msg( + <<"Health Check Interval">>, + HealthCheckInterval, + ?HEALTH_CHECK_INTERVAL_RANGE_MIN, + ?HEALTH_CHECK_INTERVAL_RANGE_MAX + ), + {error, Message}. start_after_created(type) -> boolean(); start_after_created(desc) -> ?DESC("start_after_created"); @@ -128,12 +130,14 @@ auto_restart_interval_range(AutoRestartInterval) when AutoRestartInterval =< ?AUTO_RESTART_INTERVAL_RANGE_MAX -> ok; -auto_restart_interval_range(_AutoRestartInterval) -> - {error, #{ - msg => <<"Auto Restart Interval out of range">>, - min => ?AUTO_RESTART_INTERVAL_RANGE_MIN, - max => ?AUTO_RESTART_INTERVAL_RANGE_MAX - }}. +auto_restart_interval_range(AutoRestartInterval) -> + Message = get_out_of_range_msg( + <<"Auto Restart Interval">>, + AutoRestartInterval, + ?AUTO_RESTART_INTERVAL_RANGE_MIN, + ?AUTO_RESTART_INTERVAL_RANGE_MAX + ), + {error, Message}. query_mode(type) -> enum([sync, async]); query_mode(desc) -> ?DESC("query_mode"); @@ -200,3 +204,15 @@ buffer_seg_bytes(importance) -> ?IMPORTANCE_HIDDEN; buffer_seg_bytes(_) -> undefined. desc("creation_opts") -> ?DESC("creation_opts"). + +get_value_with_unit(Value) when is_integer(Value) -> + <<(erlang:integer_to_binary(Value))/binary, "ms">>; +get_value_with_unit(Value) -> + Value. + +get_out_of_range_msg(Field, Value, Min, Max) -> + ValueStr = get_value_with_unit(Value), + MinStr = get_value_with_unit(Min), + MaxStr = get_value_with_unit(Max), + <>. From 71b636e3213b06af38a915d123c60e15e03c7a40 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 25 May 2023 12:04:23 +0800 Subject: [PATCH 07/20] fix: fix auto_restart_interval checker --- apps/emqx_resource/include/emqx_resource.hrl | 2 +- apps/emqx_resource/src/emqx_resource_manager.erl | 2 ++ apps/emqx_resource/src/schema/emqx_resource_schema.erl | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index e6f86fb59..24d606c02 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -66,7 +66,7 @@ start_after_created => boolean(), %% If the resource disconnected, we can set to retry starting the resource %% periodically. - auto_restart_interval => pos_integer(), + auto_restart_interval => pos_integer() | infinity, batch_size => pos_integer(), batch_time => pos_integer(), max_buffer_bytes => pos_integer(), diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 97ac355f4..6382a53a8 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -450,6 +450,8 @@ retry_actions(Data) -> case maps:get(auto_restart_interval, Data#data.opts, ?AUTO_RESTART_INTERVAL) of undefined -> []; + infinity -> + []; RetryInterval -> [{state_timeout, RetryInterval, auto_retry}] end. diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 48c6a3012..2dd2eda99 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -124,6 +124,8 @@ auto_restart_interval(required) -> false; auto_restart_interval(validator) -> fun auto_restart_interval_range/1; auto_restart_interval(_) -> undefined. +auto_restart_interval_range(infinity) -> + ok; auto_restart_interval_range(AutoRestartInterval) when is_integer(AutoRestartInterval) andalso AutoRestartInterval >= ?AUTO_RESTART_INTERVAL_RANGE_MIN andalso From de7f1c8aec074dde8406e293f730b10347583fa3 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 25 May 2023 17:15:19 +0800 Subject: [PATCH 08/20] test: add tests for auto_restart_interval --- .../test/emqx_bridge_api_SUITE.erl | 86 ++++++++++++++++++- .../src/emqx_resource_manager.erl | 2 + changes/ce/fix-10817.en.md | 1 + 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 changes/ce/fix-10817.en.md diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 86f088855..3c68baa0b 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -79,7 +79,8 @@ groups() -> SingleOnlyTests = [ t_broken_bpapi_vsn, t_old_bpapi_vsn, - t_bridges_probe + t_bridges_probe, + t_auto_restart_interval ], ClusterLaterJoinOnlyTCs = [t_cluster_later_join_metrics], [ @@ -550,6 +551,89 @@ t_http_crud_apis(Config) -> {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config). +t_auto_restart_interval(Config) -> + Port = ?config(port, Config), + %% assert we there's no bridges at first + {ok, 200, []} = request_json(get, uri(["bridges"]), Config), + + meck:new(emqx_resource, [passthrough]), + meck:expect(emqx_resource, call_start, fun(_, _, _) -> {error, fake_error} end), + + %% then we add a webhook bridge, using POST + %% POST /bridges/ will create a bridge + URL1 = ?URL(Port, "path1"), + Name = ?BRIDGE_NAME, + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), + BridgeParams = ?HTTP_BRIDGE(URL1, Name)#{ + <<"resource_opts">> => #{<<"auto_restart_interval">> => "1s"} + }, + ?check_trace( + begin + ?assertMatch( + {ok, 201, #{ + <<"type">> := ?BRIDGE_TYPE_HTTP, + <<"name">> := Name, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _], + <<"url">> := URL1 + }}, + request_json( + post, + uri(["bridges"]), + BridgeParams, + Config + ) + ), + ?block_until(#{?snk_kind := resource_disconnected_enter}), + ?block_until(#{?snk_kind := resource_auto_reconnect}, 1500) + end, + fun(Trace0) -> + Trace = ?of_kind(resource_auto_reconnect, Trace0), + ?assertMatch([#{}], Trace), + ok + end + ), + %% delete the bridge + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config), + + %% auto_retry_interval=infinity + BridgeParams1 = BridgeParams#{ + <<"resource_opts">> => #{<<"auto_restart_interval">> => "infinity"} + }, + ?check_trace( + begin + ?assertMatch( + {ok, 201, #{ + <<"type">> := ?BRIDGE_TYPE_HTTP, + <<"name">> := Name, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _], + <<"url">> := URL1 + }}, + request_json( + post, + uri(["bridges"]), + BridgeParams1, + Config + ) + ), + ?block_until(#{?snk_kind := resource_disconnected_enter}), + ?block_until(#{?snk_kind := resource_auto_reconnect}, 1500) + end, + fun(Trace0) -> + Trace = ?of_kind(resource_auto_reconnect, Trace0), + ?assertMatch([], Trace), + ok + end + ), + %% delete the bridge + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config), + meck:unload(emqx_resource). + t_http_bridges_local_topic(Config) -> Port = ?config(port, Config), %% assert we there's no bridges at first diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 6382a53a8..6d70422cf 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -389,8 +389,10 @@ handle_event(state_timeout, health_check, connected, Data) -> %% State: DISCONNECTED handle_event(enter, _OldState, disconnected = State, Data) -> ok = log_state_consistency(State, Data), + ?tp(resource_disconnected_enter, #{}), {keep_state_and_data, retry_actions(Data)}; handle_event(state_timeout, auto_retry, disconnected, Data) -> + ?tp(resource_auto_reconnect, #{}), start_resource(Data, undefined); %% State: STOPPED %% The stopped state is entered after the resource has been explicitly stopped diff --git a/changes/ce/fix-10817.en.md b/changes/ce/fix-10817.en.md new file mode 100644 index 000000000..f377269cb --- /dev/null +++ b/changes/ce/fix-10817.en.md @@ -0,0 +1 @@ +Fix the error of not being able to configure `auto_restart_interval` as infinity From 6b5ff24952e5cb65a9ce1d7088fca2b5c45cf655 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 25 May 2023 18:12:38 +0800 Subject: [PATCH 09/20] test: assert the block_until result --- apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 3c68baa0b..b80efdae4 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -585,8 +585,8 @@ t_auto_restart_interval(Config) -> Config ) ), - ?block_until(#{?snk_kind := resource_disconnected_enter}), - ?block_until(#{?snk_kind := resource_auto_reconnect}, 1500) + {ok, _} = ?block_until(#{?snk_kind := resource_disconnected_enter}), + {ok, _} = ?block_until(#{?snk_kind := resource_auto_reconnect}, 1500) end, fun(Trace0) -> Trace = ?of_kind(resource_auto_reconnect, Trace0), @@ -620,8 +620,8 @@ t_auto_restart_interval(Config) -> Config ) ), - ?block_until(#{?snk_kind := resource_disconnected_enter}), - ?block_until(#{?snk_kind := resource_auto_reconnect}, 1500) + {ok, _} = ?block_until(#{?snk_kind := resource_disconnected_enter}), + ?assertEqual(timeout, ?block_until(#{?snk_kind := resource_auto_reconnect}, 1500)) end, fun(Trace0) -> Trace = ?of_kind(resource_auto_reconnect, Trace0), From 194faefc8a135b210f91955aa0142ba57ad3c182 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 25 May 2023 10:52:34 -0300 Subject: [PATCH 10/20] fix(webhook): keep `resource_opts.request_timeout` for webhook bridge (r5.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes https://emqx.atlassian.net/browse/EMQX-10001 Recently, we unified request_timeout in a single field located at the webhook connector schema. However, the correct fix would be to use the resource_opts.request_timeout one, as that’s the only one that allows infinity timeout. --- apps/emqx_bridge/src/emqx_bridge_api.erl | 3 -- apps/emqx_bridge/src/emqx_bridge_resource.erl | 7 +++-- .../src/schema/emqx_bridge_schema.erl | 23 ++------------- .../src/schema/emqx_bridge_webhook_schema.erl | 29 +++++-------------- .../test/emqx_bridge_api_SUITE.erl | 14 +++++---- .../emqx_bridge_compatible_config_tests.erl | 6 ++-- .../src/emqx_connector_http.erl | 3 +- 7 files changed, 27 insertions(+), 58 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 847270664..dde2b64c9 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -915,9 +915,6 @@ fill_defaults(Type, RawConf) -> pack_bridge_conf(Type, RawConf) -> #{<<"bridges">> => #{bin(Type) => #{<<"foo">> => RawConf}}}. -%% Hide webhook's resource_opts.request_timeout from user. -filter_raw_conf(<<"webhook">>, RawConf0) -> - emqx_utils_maps:deep_remove([<<"resource_opts">>, <<"request_timeout">>], RawConf0); filter_raw_conf(_TypeBin, RawConf) -> RawConf. diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index e622403b3..fd67c622c 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -311,7 +311,6 @@ parse_confs( url := Url, method := Method, headers := Headers, - request_timeout := ReqTimeout, max_retries := Retry } = Conf ) -> @@ -325,6 +324,10 @@ parse_confs( Reason1 = emqx_utils:readable_error_msg(Reason), invalid_data(<<"Invalid URL: ", Url1/binary, ", details: ", Reason1/binary>>) end, + RequestTimeout = emqx_utils_maps:deep_get( + [resource_opts, request_timeout], + Conf + ), Conf#{ base_url => BaseUrl1, request => @@ -333,7 +336,7 @@ parse_confs( method => Method, body => maps:get(body, Conf, undefined), headers => Headers, - request_timeout => ReqTimeout, + request_timeout => RequestTimeout, max_retries => Retry } }; diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 90e62764a..03ae781ca 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -251,25 +251,6 @@ node_name() -> {"node", mk(binary(), #{desc => ?DESC("desc_node_name"), example => "emqx@127.0.0.1"})}. webhook_bridge_converter(Conf0, _HoconOpts) -> - Conf1 = emqx_bridge_compatible_config:upgrade_pre_ee( + emqx_bridge_compatible_config:upgrade_pre_ee( Conf0, fun emqx_bridge_compatible_config:webhook_maybe_upgrade/1 - ), - case Conf1 of - undefined -> - undefined; - _ -> - maps:map( - fun(_Name, Conf) -> - do_convert_webhook_config(Conf) - end, - Conf1 - ) - end. - -%% We hide resource_opts.request_timeout from user. -do_convert_webhook_config( - #{<<"request_timeout">> := ReqT, <<"resource_opts">> := ResOpts} = Conf -) -> - Conf#{<<"resource_opts">> => ResOpts#{<<"request_timeout">> => ReqT}}; -do_convert_webhook_config(Conf) -> - Conf. + ). diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl index de1e09abb..4e1492a9c 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl @@ -40,15 +40,12 @@ fields("put") -> fields("get") -> emqx_bridge_schema:status_fields() ++ fields("post"); fields("creation_opts") -> - [ - hidden_request_timeout() - | lists:filter( - fun({K, _V}) -> - not lists:member(K, unsupported_opts()) - end, - emqx_resource_schema:fields("creation_opts") - ) - ]. + lists:filter( + fun({K, _V}) -> + not lists:member(K, unsupported_opts()) + end, + emqx_resource_schema:fields("creation_opts") + ). desc("config") -> ?DESC("desc_config"); @@ -144,6 +141,7 @@ request_config() -> emqx_schema:duration_ms(), #{ default => <<"15s">>, + deprecated => {since, "v5.0.26"}, desc => ?DESC("config_request_timeout") } )} @@ -166,8 +164,7 @@ unsupported_opts() -> [ enable_batch, batch_size, - batch_time, - request_timeout + batch_time ]. %%====================================================================================== @@ -194,13 +191,3 @@ name_field() -> method() -> enum([post, put, get, delete]). - -hidden_request_timeout() -> - {request_timeout, - mk( - hoconsc:union([infinity, emqx_schema:duration_ms()]), - #{ - required => false, - importance => ?IMPORTANCE_HIDDEN - } - )}. diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index b80efdae4..a902783d7 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -1390,18 +1390,20 @@ t_inconsistent_webhook_request_timeouts(Config) -> <<"resource_opts">> => #{<<"request_timeout">> => <<"2s">>} } ), - {ok, 201, #{ - <<"request_timeout">> := <<"1s">>, - <<"resource_opts">> := ResourceOpts - }} = + %% root request_timeout is deprecated for bridge. + {ok, 201, + #{ + <<"resource_opts">> := ResourceOpts + } = Response} = request_json( post, uri(["bridges"]), BadBridgeParams, Config ), - ?assertNot(maps:is_key(<<"request_timeout">>, ResourceOpts)), - validate_resource_request_timeout(proplists:get_value(group, Config), 1000, Name), + ?assertNot(maps:is_key(<<"request_timeout">>, Response)), + ?assertMatch(#{<<"request_timeout">> := <<"2s">>}, ResourceOpts), + validate_resource_request_timeout(proplists:get_value(group, Config), 2000, Name), ok. t_cluster_later_join_metrics(Config) -> diff --git a/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl b/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl index 080ca9107..af9daef0a 100644 --- a/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl @@ -65,15 +65,13 @@ webhook_config_test() -> <<"the_name">> := #{ <<"method">> := get, - <<"request_timeout">> := RequestTime, <<"resource_opts">> := ResourceOpts, <<"body">> := <<"${payload}">> } } } } = check(Conf3), - ?assertEqual(60_000, RequestTime), - ?assertMatch(#{<<"request_timeout">> := 60_000}, ResourceOpts), + ?assertMatch(#{<<"request_timeout">> := infinity}, ResourceOpts), ok. up(#{<<"bridges">> := Bridges0} = Conf0) -> @@ -196,7 +194,7 @@ full_webhook_v5019_hocon() -> " pool_type = \"random\"\n" " request_timeout = \"1m\"\n" " resource_opts = {\n" - " request_timeout = \"7s\"\n" + " request_timeout = \"infinity\"\n" " }\n" " ssl {\n" " ciphers = \"\"\n" diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 4a9cb345f..cce62b410 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -58,6 +58,7 @@ ]). -define(DEFAULT_PIPELINE_SIZE, 100). +-define(DEFAULT_REQUEST_TIMEOUT_MS, 30_000). %%===================================================================== %% Hocon schema @@ -477,7 +478,7 @@ preprocess_request( path => emqx_plugin_libs_rule:preproc_tmpl(Path), body => maybe_preproc_tmpl(body, Req), headers => wrap_auth_header(preproc_headers(Headers)), - request_timeout => maps:get(request_timeout, Req, 30000), + request_timeout => maps:get(request_timeout, Req, ?DEFAULT_REQUEST_TIMEOUT_MS), max_retries => maps:get(max_retries, Req, 2) }. From ea86f4442b966b9487ef3749e2f2da05ee510ffa Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Fri, 19 May 2023 20:16:05 -0300 Subject: [PATCH 11/20] fix: avoid error 500 when node is re-joining cluster Fixes https://emqx.atlassian.net/browse/EMQX-9899 --- apps/emqx/src/emqx_rpc.erl | 2 +- apps/emqx_authz/src/emqx_authz_api_mnesia.erl | 8 ++-- apps/emqx_bridge/src/emqx_bridge_api.erl | 9 ++++- apps/emqx_ctl/src/emqx_ctl.app.src | 2 +- apps/emqx_ctl/src/emqx_ctl.erl | 2 +- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- .../src/emqx_gateway_api_clients.erl | 6 ++- apps/emqx_management/src/emqx_mgmt_api.erl | 40 +++++++++++++------ .../src/emqx_mgmt_api_alarms.erl | 4 +- .../src/emqx_mgmt_api_topics.erl | 4 +- .../src/emqx_rule_engine_api.erl | 3 ++ changes/ce/fix-10760.en.md | 1 + 12 files changed, 55 insertions(+), 28 deletions(-) create mode 100644 changes/ce/fix-10760.en.md diff --git a/apps/emqx/src/emqx_rpc.erl b/apps/emqx/src/emqx_rpc.erl index 062bde68b..86ec5937f 100644 --- a/apps/emqx/src/emqx_rpc.erl +++ b/apps/emqx/src/emqx_rpc.erl @@ -147,7 +147,7 @@ unwrap_erpc({throw, A}) -> {error, A}; unwrap_erpc({error, {exception, Err, _Stack}}) -> {error, Err}; -unwrap_erpc({error, {exit, Err}}) -> +unwrap_erpc({exit, Err}) -> {error, Err}; unwrap_erpc({error, {erpc, Err}}) -> {error, Err}. diff --git a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl index b39379b43..f5ac40f5e 100644 --- a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl @@ -423,8 +423,8 @@ users(get, #{query_string := QueryString}) -> of {error, page_limit_invalid} -> {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}}; - {error, Node, {badrpc, R}} -> - Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, R])), + {error, Node, Error} -> + Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error])), {500, #{code => <<"NODE_DOWN">>, message => Message}}; Result -> {200, Result} @@ -459,8 +459,8 @@ clients(get, #{query_string := QueryString}) -> of {error, page_limit_invalid} -> {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}}; - {error, Node, {badrpc, R}} -> - Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, R])), + {error, Node, Error} -> + Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error])), {500, #{code => <<"NODE_DOWN">>, message => Message}}; Result -> {200, Result} diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 847270664..026c24ae2 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -756,7 +756,14 @@ format_bridge_info([FirstBridge | _] = Bridges) -> }). format_bridge_metrics(Bridges) -> - NodeMetrics = collect_metrics(Bridges), + FilteredBridges = lists:filter( + fun + ({_Node, Metric}) when is_map(Metric) -> true; + (_) -> false + end, + Bridges + ), + NodeMetrics = collect_metrics(FilteredBridges), #{ metrics => aggregate_metrics(NodeMetrics), node_metrics => NodeMetrics diff --git a/apps/emqx_ctl/src/emqx_ctl.app.src b/apps/emqx_ctl/src/emqx_ctl.app.src index 9de598a89..c3abade67 100644 --- a/apps/emqx_ctl/src/emqx_ctl.app.src +++ b/apps/emqx_ctl/src/emqx_ctl.app.src @@ -1,6 +1,6 @@ {application, emqx_ctl, [ {description, "Backend for emqx_ctl script"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_ctl_app, []}}, {applications, [ diff --git a/apps/emqx_ctl/src/emqx_ctl.erl b/apps/emqx_ctl/src/emqx_ctl.erl index 864b53d2a..6123056b9 100644 --- a/apps/emqx_ctl/src/emqx_ctl.erl +++ b/apps/emqx_ctl/src/emqx_ctl.erl @@ -228,7 +228,7 @@ handle_call({register_command, Cmd, MF, Opts}, _From, State = #state{seq = Seq}) ets:insert(?CMD_TAB, {{Seq, Cmd}, MF, Opts}), {reply, ok, next_seq(State)}; [[OriginSeq] | _] -> - ?LOG_WARNING(#{msg => "CMD_overidden", cmd => Cmd, mf => MF}), + ?LOG_WARNING(#{msg => "CMD_overridden", cmd => Cmd, mf => MF}), true = ets:insert(?CMD_TAB, {{OriginSeq, Cmd}, MF, Opts}), {reply, ok, State} end; diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 2ffca464d..26dab4c9c 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.16"}, + {vsn, "0.1.17"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, emqx, emqx_authn, emqx_ctl]}, diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index cd387e3bb..7f8f30c8b 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -133,8 +133,10 @@ clients(get, #{ case Result of {error, page_limit_invalid} -> {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}}; - {error, Node, {badrpc, R}} -> - Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, R])), + {error, Node, Error} -> + Message = list_to_binary( + io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error]) + ), {500, #{code => <<"NODE_DOWN">>, message => Message}}; Response -> {200, Response} diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index 8365b983c..7280c17b6 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -134,8 +134,8 @@ do_node_query( ResultAcc ) -> case do_query(Node, QueryState) of - {error, {badrpc, R}} -> - {error, Node, {badrpc, R}}; + {error, Error} -> + {error, Node, Error}; {Rows, NQueryState = #{complete := Complete}} -> case accumulate_query_rows(Node, Rows, NQueryState, ResultAcc) of {enough, NResultAcc} -> @@ -179,8 +179,8 @@ do_cluster_query( ResultAcc ) -> case do_query(Node, QueryState) of - {error, {badrpc, R}} -> - {error, Node, {badrpc, R}}; + {error, Error} -> + {error, Node, Error}; {Rows, NQueryState = #{complete := Complete}} -> case accumulate_query_rows(Node, Rows, NQueryState, ResultAcc) of {enough, NResultAcc} -> @@ -275,7 +275,7 @@ do_query(Node, QueryState) when Node =:= node() -> do_select(Node, QueryState); do_query(Node, QueryState) -> case - rpc:call( + catch rpc:call( Node, ?MODULE, do_query, @@ -284,6 +284,7 @@ do_query(Node, QueryState) -> ) of {badrpc, _} = R -> {error, R}; + {'EXIT', _} = R -> {error, R}; Ret -> Ret end. @@ -298,15 +299,24 @@ do_select( ) -> QueryState = maybe_apply_total_query(Node, QueryState0), Result = - case maps:get(continuation, QueryState, undefined) of - undefined -> - ets:select(Tab, Ms, Limit); - Continuation -> - %% XXX: Repair is necessary because we pass Continuation back - %% and forth through the nodes in the `do_cluster_query` - ets:select(ets:repair_continuation(Continuation, Ms)) + try + case maps:get(continuation, QueryState, undefined) of + undefined -> + ets:select(Tab, Ms, Limit); + Continuation -> + %% XXX: Repair is necessary because we pass Continuation back + %% and forth through the nodes in the `do_cluster_query` + ets:select(ets:repair_continuation(Continuation, Ms)) + end + catch + exit:_ = Exit -> + {error, Exit}; + Type:Reason:Stack -> + {error, #{exception => Type, reason => Reason, stacktrace => Stack}} end, case Result of + {error, _} -> + {[], mark_complete(QueryState)}; {Rows, '$end_of_table'} -> NRows = maybe_apply_fuzzy_filter(Rows, QueryState), {NRows, mark_complete(QueryState)}; @@ -354,7 +364,11 @@ counting_total_fun(_QueryState = #{match_spec := Ms, fuzzy_fun := undefined}) -> [{MatchHead, Conditions, _Return}] = Ms, CountingMs = [{MatchHead, Conditions, [true]}], fun(Tab) -> - ets:select_count(Tab, CountingMs) + try + ets:select_count(Tab, CountingMs) + catch + _Type:_Reason -> 0 + end end; counting_total_fun(_QueryState = #{fuzzy_fun := FuzzyFun}) when FuzzyFun =/= undefined -> %% XXX: Calculating the total number for a fuzzy searching is very very expensive diff --git a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl index 88c2be89d..d5965f019 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl @@ -123,8 +123,8 @@ alarms(get, #{query_string := QString}) -> of {error, page_limit_invalid} -> {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}}; - {error, Node, {badrpc, R}} -> - Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, R])), + {error, Node, Error} -> + Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error])), {500, #{code => <<"NODE_DOWN">>, message => Message}}; Response -> {200, Response} diff --git a/apps/emqx_management/src/emqx_mgmt_api_topics.erl b/apps/emqx_management/src/emqx_mgmt_api_topics.erl index d451261ff..f6e790bf4 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_topics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_topics.erl @@ -120,8 +120,8 @@ do_list(Params) -> of {error, page_limit_invalid} -> {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}}; - {error, Node, {badrpc, R}} -> - Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, R])), + {error, Node, Error} -> + Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error])), {500, #{code => <<"NODE_DOWN">>, message => Message}}; Response -> {200, Response} diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index d66f2c1c9..d36ec6a40 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -339,6 +339,9 @@ param_path_id() -> of {error, page_limit_invalid} -> {400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}}; + {error, Node, Error} -> + Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error])), + {500, #{code => <<"NODE_DOWN">>, message => Message}}; Result -> {200, Result} end; diff --git a/changes/ce/fix-10760.en.md b/changes/ce/fix-10760.en.md new file mode 100644 index 000000000..42c71e66a --- /dev/null +++ b/changes/ce/fix-10760.en.md @@ -0,0 +1 @@ +Fix Internal Error 500 that occurred sometimes when bridge statistics page was updated while a node was (re)joining the cluster. From ed7db7369a77ac6b4f93b0c6a4aa5172a5721001 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Fri, 26 May 2023 16:58:14 +0800 Subject: [PATCH 12/20] chore: upgrade dashboard to e1.0.7 for ee and v1.2.5 for ce --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 785458601..b579a078d 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,8 @@ endif # Dashbord version # from https://github.com/emqx/emqx-dashboard5 -export EMQX_DASHBOARD_VERSION ?= v1.2.4-1 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.7-beta.3 +export EMQX_DASHBOARD_VERSION ?= v1.2.5 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.7 # `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used # In make 4.4+, for backward-compatibility the value from the original environment is used. From 2d986140df294501578be856b2908cb021aff076 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 26 May 2023 12:01:10 +0200 Subject: [PATCH 13/20] chore: bump version to e5.0.4-rc.2 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index b23e59f29..701c75178 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.25"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.4-alpha.2"). +-define(EMQX_RELEASE_EE, "5.0.4-rc.1"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). From 8941b6c287ca5972544e899497c11e00a87eb751 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 26 May 2023 12:25:58 +0200 Subject: [PATCH 14/20] docs: add merged change logs e5.0.4.en.md --- changes/e5.0.4.en.md | 167 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 changes/e5.0.4.en.md diff --git a/changes/e5.0.4.en.md b/changes/e5.0.4.en.md new file mode 100644 index 000000000..8f699bc11 --- /dev/null +++ b/changes/e5.0.4.en.md @@ -0,0 +1,167 @@ +# e5.0.4 + +## Enhancements + +- [#10389](https://github.com/emqx/emqx/pull/10389) Unified the configuration formats for `cluster.core_nodes` and `cluster.statics.seeds`. Now they both support formats in array `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` and the comma-separated string `"emqx1@127.0.0.1,emqx2@127.0.0.1"`. + +- [#10392](https://github.com/emqx/emqx/pull/10392) Introduced a new function to convert a formatted date to an integer timestamp: date_to_unix_ts/3. + + `date_to_unix_ts(TimeUnit, FormatString, InputDateTimeString)` + +- [#10426](https://github.com/emqx/emqx/pull/10426) Optimized the configuration priority mechanism to fix the issue where the configuration changes made to `etc/emqx.conf` do not take effect after restarting EMQX. + + More information about the new mechanism: [Configure Override Rules](https://www.emqx.io/docs/en/v5.0/configuration/configuration.html#configure-override-rules) + +- [#10457](https://github.com/emqx/emqx/pull/10457) Deprecated the integration with StatsD. + +- [#10458](https://github.com/emqx/emqx/pull/10458) Set the level of plugin configuration options to low, users usually manage the plugins through the dashboard, rarely modify them manually, so we lowered the level. + +- [#10491](https://github.com/emqx/emqx/pull/10491) Renamed `etcd.ssl` to `etcd.ssl_options` to keep all SSL options consistent in the configuration file. + +- [#10512](https://github.com/emqx/emqx/pull/10512) Improved the storage format of Unicode characters in data files, Now we can store Unicode characters. For example: `SELECT * FROM "t/1" WHERE clientid = "-测试专用-"`. + +- [#10568](https://github.com/emqx/emqx/pull/10568) Added `shutdown_counter` printout to `emqx ctl listeners` command. + +- [#10588](https://github.com/emqx/emqx/pull/10588) Increased the time precision of trace logs from second to microsecond. For example, change from `2023-05-02T08:43:50+00:00` to `2023-05-02T08:43:50.237945+00:00`. + +- [#10623](https://github.com/emqx/emqx/pull/10623) Renamed `max_message_queue_len` to `max_mailbox_size` in the `force_shutdown` configuration. The old name is kept as an alias, so this change is backward compatible. + +- [#10713](https://github.com/emqx/emqx/pull/10713) Hide the `resource_option.request_timeout` of the webhook and it will use the value of `http` `request_timeout`. + +- [#10075](https://github.com/emqx/emqx/pull/10075) Added node rebalance/node evacuation functionality. See also: [EIP doc](https://github.com/emqx/eip/blob/main/active/0020-node-rebalance.md) + +- [#10378](https://github.com/emqx/emqx/pull/10378) Implemented Pulsar Producer Bridge and only producer role is supported now. + +- [#10408](https://github.com/emqx/emqx/pull/10408) Introduced 3 built-in functions in the rule engine SQL-like language for creating values of the MongoDB date type. + +- [#10409](https://github.com/emqx/emqx/pull/10409) [#10337](#10337) Supported [Protocol Buffers](https://protobuf.dev/) and [Apache Avro](https://avro.apache.org/) schemas in Schema Registry. + +- [#10425](https://github.com/emqx/emqx/pull/10425) Implemented OpenTSDB data bridge. + +- [#10498](https://github.com/emqx/emqx/pull/10498) Implemented Oracle Database Bridge. + +- [#10560](https://github.com/emqx/emqx/pull/10560) Added enterprise data bridge for Apache IoTDB. + +- [#10417](https://github.com/emqx/emqx/pull/10417) Improved get config items performance by eliminating temporary references. + +- [#10430](https://github.com/emqx/emqx/pull/10430) Simplified the configuration of the `retainer` feature. Marked `flow_control` as a non-importance field. + +- [#10511](https://github.com/emqx/emqx/pull/10511) Improved the security and privacy of some resource logs by masking sensitive information in the log. + +- [#10525](https://github.com/emqx/emqx/pull/10525) Reduced resource usage per MQTT packet handling. + +- [#10528](https://github.com/emqx/emqx/pull/10528) Reduced memory footprint in hot code path. The hot path includes the code that is frequently executed in core functionalities such as message handling, connection management, authentication, and authorization. + +- [#10591](https://github.com/emqx/emqx/pull/10591) [#10625](https://github.com/emqx/emqx/pull/10625) Improved the configuration of the limiter. + + - Reduced the complexity of the limiter's configuration. + + - Updated the `configs/limiter` API to suit this refactor. + + - Reduced the memory usage of the limiter configuration. + +- [#10487](https://github.com/emqx/emqx/pull/10487) Optimized the instance of limiter for whose rate is `infinity` to reduce memory and CPU usage. + +- [#10490](https://github.com/emqx/emqx/pull/10490) Removed the default limit of connect rate which used to be `1000/s`. + +- [#10077](https://github.com/emqx/emqx/pull/10077) Added support for QUIC TLS password-protected certificate file. + +## Bug Fixes + +- [#10340](https://github.com/emqx/emqx/pull/10340) Fixed the issue that could lead to crash logs being printed when stopping EMQX via `systemd`. + + `2023-03-29T16:43:25.915761+08:00 [error] Generic server memsup terminating. Reason: {port_died,normal}. Last message: {'EXIT',<0.2117.0>,{port_died,normal}}. State: [{data,[{"Timeout",60000}]},{items,{"Memory Usage",[{"Allocated",929959936},{"Total",3832242176}]}},{items,{"Worst Memory User",[{"Pid",<0.2031.0>},{"Memory",4720472}]}}]. 2023-03-29T16:43:25.924764+08:00 [error] crasher: initial call: memsup:init/1, pid: <0.2116.0>, registered_name: memsup, exit: {{port_died,normal},[{gen_server,handle_common_reply,8,[{file,"gen_server.erl"},{line,811}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}]}, ancestors: [os_mon_sup,<0.2114.0>], message_queue_len: 0, messages: [], links: [<0.2115.0>], dictionary: [], trap_exit: true, status: running, heap_size: 4185, stack_size: 29, reductions: 187637; neighbours: 2023-03-29T16:43:25.924979+08:00 [error] Supervisor: {local,os_mon_sup}. Context: child_terminated. Reason: {port_died,normal}. Offender: id=memsup,pid=<0.2116.0>.` + +- [#10369](https://github.com/emqx/emqx/pull/10369) Fixed error in `/api/v5/monitor_current` API endpoint that happens when some EMQX nodes are down. + + Prior to this fix, sometimes the request returned HTTP code 500 and the following message: + + `{"code":"INTERNAL_ERROR","message":"error, badarg, [{erlang,'++',[{error,nodedown},[{node,'emqx@10.42.0.150'}]], ...` + +- [#10407](https://github.com/emqx/emqx/pull/10407) Fixed the crash issue of the alarm system. + + - Leverage Mnesia dirty operations and circumvent extraneous calls to enhance 'emqx_alarm' performance. + + - Use 'emqx_resource_manager' for reactivating alarms that have already been triggered. + + - Implement the newly developed, fail-safe 'emqx_alarm' API to control the activation and deactivation of alarms, thus preventing 'emqx_resource_manager' from crashing due to alarm timeouts. + + - The alarm system is susceptible to crashing under these concurrent conditions: + + - A significant number of resources fail, such as when bridges continuously attempt to trigger alarms due to recurring errors. + + - The system is under an extremely high load. + +- [#10420](https://github.com/emqx/emqx/pull/10420) Fixed HTTP path handling when composing the URL for the HTTP requests in authentication and authorization modules. + + - Avoid unnecessary URL normalization since we cannot assume that external servers treat original and normalized URLs equally. This led to bugs like [#10411](https://github.com/emqx/emqx/issues/10411). + + - Fixed the issue that path segments could be HTTP encoded twice. + +- [#10422](https://github.com/emqx/emqx/pull/10422) Fixed a bug where external plugins could not be configured via environment variables in a lone-node cluster. + +- [#10448](https://github.com/emqx/emqx/pull/10448) Fixed a compatibility issue of limiter configuration introduced by e5.0.3 which broke the upgrade from previous versions if the `capacity` is `infinity`. + + In e5.0.3 we have replaced `capacity` with `burst`. After this fix, a `capacity = infinity` config will be automatically converted to equivalent `burst = 0`. + +- [#10462](https://github.com/emqx/emqx/pull/10462) Deprecated config `broker.shared_dispatch_ack_enabled`. This was designed to avoid dispatching messages to a shared-subscription session that has the client disconnected. However, since e5.0.0, this feature is no longer helpful because the shared-subscription messages in an expired session will be redispatched to other sessions in the group. See also: . + +- [#10463](https://github.com/emqx/emqx/pull/10463) Improved bridges API error handling. If Webhook bridge URL is not valid, the bridges API will return '400' error instead of '500'. + +- [#10484](https://github.com/emqx/emqx/pull/10484) Fixed the issue that the priority of the configuration cannot be set during the rolling upgrade. For example, when authorization is modified in e5.0.2 and then upgraded e5.0.3 through the rolling upgrade, the authorization will be restored to the default. + +- [#10495](https://github.com/emqx/emqx/pull/10495) Added the limiter API `/configs/limiter` which was deleted by mistake back. + +- [#10500](https://github.com/emqx/emqx/pull/10500) Added several fixes, enhancements, and features in Mria: + + - Protect `mria:join/1,2` with a global lock to prevent conflicts between two nodes trying to join each other simultaneously [Mria PR](https://github.com/emqx/mria/pull/137) + + - Implement new function `mria:sync_transaction/4,3,2`, which blocks the caller until a transaction is imported to the local node (if the local node is a replicant, otherwise, it behaves exactly the same as `mria:transaction/3,2`) [Mria PR](https://github.com/emqx/mria/pull/136) + + - Optimize `mria:running_nodes/0` [Mria PR](https://github.com/emqx/mria/pull/135) + + - Optimize `mria:ro_transaction/2` when called on a replicant node [Mria PR](https://github.com/emqx/mria/pull/134). + +- [#10518](https://github.com/emqx/emqx/pull/10518) Added the following fixes and features in Mria: + + - Call `mria_rlog:role/1` safely in mria_membership to ensure that mria_membership gen_server won't crash if RPC to another node fails [Mria PR](https://github.com/emqx/mria/pull/139) + + - Add an extra field to `?rlog_sync` table to facilitate extending this functionality in future [Mria PR](https://github.com/emqx/mria/pull/138). + +- [#10556](https://github.com/emqx/emqx/pull/10556) Wrapped potentially sensitive data in `emqx_connector_http` if `Authorization` headers are being passed at initialization. + +- [#10571](https://github.com/emqx/emqx/pull/10571) Stopped emitting useless crash report when EMQX stops. + +- [#10659](https://github.com/emqx/emqx/pull/10659) Fixed the issue where EMQX cannot start when `sysmon.os.mem_check_interval` is disabled. + +- [#10717](https://github.com/emqx/emqx/pull/10717) Fixed an issue where the buffering layer processes could use a lot of CPU when inflight window is full. + +- [#10724](https://github.com/emqx/emqx/pull/10724) A summary has been added for all endpoints in the HTTP API documentation (accessible at "http://:18083/api-docs"). + +- [#10726](https://github.com/emqx/emqx/pull/10726) Health Check Interval and Auto Restart Interval now support the range from 1ms to 1 hour. + +- [#10728](https://github.com/emqx/emqx/pull/10728) Fixed an issue where the rule engine was unable to access variables exported by `FOREACH` - `DO` clause. + + Given a payload: `{"date": "2023-05-06", "array": ["a"]}`, as well as the following SQL statement: + + `FOREACH payload.date as date, payload.array as elem DO date, elem FROM "t/#" -- {"date": "2023-05-06", "array": ["a"]}` + + Prior to the fix, the `date` variable exported by `FOREACH` could not be accessed in the `DO` clause of the above SQL, resulting in the following output for the SQL statement: `[{"elem": "a","date": "undefined"}]`. + +- [#10742](https://github.com/emqx/emqx/pull/10742) Correctness check of the rules is enforced before saving the authorization file source. Previously, Saving wrong rules could lead to EMQX restart failure. + +- [#10743](https://github.com/emqx/emqx/pull/10743) Fixed an issue where trying to get bridge info or metrics could result in a crash when a node is joining a cluster. + +- [#10755](https://github.com/emqx/emqx/pull/10755) Fixed data bridge resource update race condition. + + In the 'delete + create' process for EMQX resource updates, long bridge creation times could cause dashboard request timeouts. If a bridge resource update was initiated before completion of its creation, it led to an erroneous deletion from the runtime, despite being present in the config file. + + This fix addresses the race condition in bridge resource updates, ensuring the accurate identification and addition of new resources, and maintaining consistency between runtime and configuration file statuses. + +- [#10761](https://github.com/emqx/emqx/pull/10761) Fixed the issue where the default value of SSL certificate for Dashboard Listener was not correctly interpolated, which caused HTTPS to be inaccessible when `verify_peer` and `cacertfile` were using the default configuration. + +- [#10672](https://github.com/emqx/emqx/pull/10672) Fixed the issue where the lack of a default value for `ssl_options` in listeners results in startup failure. For example, such command(`EMQX_LISTENERS__WSS__DEFAULT__BIND='0.0.0.0:8089' ./bin/emqx console`) would have caused a crash before. + +- [#10738](https://github.com/emqx/emqx/pull/10738) TDEngine data bridge now supports "Supertable" and "Create Tables Automatically". Before this fix, an insert with a supertable in the template will fail, like this: + + - `insert into ${clientid} using msg TAGS (${clientid}) values (${ts},${msg})`. From 467a2acdb870e28e9dfd272b3944fb5d6d551d07 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 26 May 2023 12:59:46 +0200 Subject: [PATCH 15/20] docs: add missing changes --- changes/e5.0.4.en.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/changes/e5.0.4.en.md b/changes/e5.0.4.en.md index 8f699bc11..40b77da1b 100644 --- a/changes/e5.0.4.en.md +++ b/changes/e5.0.4.en.md @@ -165,3 +165,13 @@ - [#10738](https://github.com/emqx/emqx/pull/10738) TDEngine data bridge now supports "Supertable" and "Create Tables Automatically". Before this fix, an insert with a supertable in the template will fail, like this: - `insert into ${clientid} using msg TAGS (${clientid}) values (${ts},${msg})`. + +- [#10746](https://github.com/emqx/emqx/pull/10746) Add missing support of the event `$events/delivery_dropped` into the rule engine test API `rule_test`. + +- [#10747](https://github.com/emqx/emqx/pull/10747) Ported some time formating fixes in Rule-Engine functions from version 4.4. + +- [#10760](https://github.com/emqx/emqx/pull/10760) Fix "internal error 500" when getting bridge statistics page while a node is joining the cluster. + +- [#10801](https://github.com/emqx/emqx/pull/10801) Avoid double percent-decode for topic name in API `/topics/{topic}` and `/topics`. + +- [#10817](https://github.com/emqx/emqx/pull/10817) Fix a config value handling for bridge resource option `auto_restart_interval`, now it can be set to `infinity`. From 7eb22e27c4e3989df0dab50dcb89719e31461538 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 26 May 2023 14:34:01 +0200 Subject: [PATCH 16/20] chore: bump release version to e5.0.4 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 701c75178..ec78091b7 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.25"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.4-rc.1"). +-define(EMQX_RELEASE_EE, "5.0.4"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). From 99b56a53d1055a96df4c78a85beafc41561c793e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 26 May 2023 14:34:31 +0200 Subject: [PATCH 17/20] docs: Generate changelog for e5.0.4 --- changes/e5.0.4.en.md | 259 +++++++++++++++++++++++++------------------ 1 file changed, 154 insertions(+), 105 deletions(-) diff --git a/changes/e5.0.4.en.md b/changes/e5.0.4.en.md index 40b77da1b..3f5c91f92 100644 --- a/changes/e5.0.4.en.md +++ b/changes/e5.0.4.en.md @@ -2,176 +2,225 @@ ## Enhancements -- [#10389](https://github.com/emqx/emqx/pull/10389) Unified the configuration formats for `cluster.core_nodes` and `cluster.statics.seeds`. Now they both support formats in array `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` and the comma-separated string `"emqx1@127.0.0.1,emqx2@127.0.0.1"`. +- [#10389](https://github.com/emqx/emqx/pull/10389) Unify the config formats for `cluster.core_nodes` and `cluster.statics.seeds`. + Now they both support formats in array `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` or semicolon-separated string `"emqx1@127.0.0.1,emqx2@127.0.0.1"`. -- [#10392](https://github.com/emqx/emqx/pull/10392) Introduced a new function to convert a formatted date to an integer timestamp: date_to_unix_ts/3. +- [#10392](https://github.com/emqx/emqx/pull/10392) A new function to convert a formatted date to an integer timestamp has been added: date_to_unix_ts/3 - `date_to_unix_ts(TimeUnit, FormatString, InputDateTimeString)` +- [#10426](https://github.com/emqx/emqx/pull/10426) Optimize the configuration priority mechanism to fix the issue where the configuration + changes made to `etc/emqx.conf` do not take effect after restarting EMQX. -- [#10426](https://github.com/emqx/emqx/pull/10426) Optimized the configuration priority mechanism to fix the issue where the configuration changes made to `etc/emqx.conf` do not take effect after restarting EMQX. + More introduction about the new mechanism: [Configure Override Rules](https://www.emqx.io/docs/en/v5.0/configuration/configuration.html#configure-override-rules) - More information about the new mechanism: [Configure Override Rules](https://www.emqx.io/docs/en/v5.0/configuration/configuration.html#configure-override-rules) +- [#10457](https://github.com/emqx/emqx/pull/10457) Deprecates the integration with StatsD. -- [#10457](https://github.com/emqx/emqx/pull/10457) Deprecated the integration with StatsD. + There seemd to be no user using StatsD integration, so we have decided to hide this feature + for now. We will either remove or revive it based on requirements in the future. -- [#10458](https://github.com/emqx/emqx/pull/10458) Set the level of plugin configuration options to low, users usually manage the plugins through the dashboard, rarely modify them manually, so we lowered the level. +- [#10458](https://github.com/emqx/emqx/pull/10458) Set the level of plugin configuration options to low level, + in most cases, users only need to manage plugins on the dashboard + without the need for manual modification, so we lowered the level. -- [#10491](https://github.com/emqx/emqx/pull/10491) Renamed `etcd.ssl` to `etcd.ssl_options` to keep all SSL options consistent in the configuration file. +- [#10491](https://github.com/emqx/emqx/pull/10491) Rename `etcd.ssl` to `etcd.ssl_options` to keep all of SSL options consistent in the configuration file. -- [#10512](https://github.com/emqx/emqx/pull/10512) Improved the storage format of Unicode characters in data files, Now we can store Unicode characters. For example: `SELECT * FROM "t/1" WHERE clientid = "-测试专用-"`. +- [#10512](https://github.com/emqx/emqx/pull/10512) Improved the storage format of Unicode characters in data files, + Now we can store Unicode characters normally. + For example: "SELECT * FROM \"t/1\" WHERE clientid = \"-测试专用-\"" -- [#10568](https://github.com/emqx/emqx/pull/10568) Added `shutdown_counter` printout to `emqx ctl listeners` command. +- [#10568](https://github.com/emqx/emqx/pull/10568) Add shutdown counter information to `emqx ctl listeners` command -- [#10588](https://github.com/emqx/emqx/pull/10588) Increased the time precision of trace logs from second to microsecond. For example, change from `2023-05-02T08:43:50+00:00` to `2023-05-02T08:43:50.237945+00:00`. +- [#10571](https://github.com/emqx/emqx/pull/10571) Do not emit useless crash report when EMQX stops. + Previously, when EMQX (and `emqx_topic_metrics` in particular) stopped and removed underlying tables, some messages were still being handled and crashed. -- [#10623](https://github.com/emqx/emqx/pull/10623) Renamed `max_message_queue_len` to `max_mailbox_size` in the `force_shutdown` configuration. The old name is kept as an alias, so this change is backward compatible. +- [#10588](https://github.com/emqx/emqx/pull/10588) Increase the time precision of trace logs from second to microsecond. + For example, change from `2023-05-02T08:43:50+00:00` to `2023-05-02T08:43:50.237945+00:00`. -- [#10713](https://github.com/emqx/emqx/pull/10713) Hide the `resource_option.request_timeout` of the webhook and it will use the value of `http` `request_timeout`. +- [#10623](https://github.com/emqx/emqx/pull/10623) Renamed `max_message_queue_len` to `max_mailbox_size` in the `force_shutdown` configuration. Old name is kept as an alias, so this change is backward compatible. -- [#10075](https://github.com/emqx/emqx/pull/10075) Added node rebalance/node evacuation functionality. See also: [EIP doc](https://github.com/emqx/eip/blob/main/active/0020-node-rebalance.md) +- [#10713](https://github.com/emqx/emqx/pull/10713) We hide the request_timeout in resource_option of the webhook to keep it consistent with the http request_timeout of the webhook. + From now on, when configuring a webhook through API or configuration files, + it is no longer necessary to configure the request_timeout of the resource. Only configuring the http request_timeout is sufficient, and the request_timeout in the resource will automatically be consistent with the http request_timeout. -- [#10378](https://github.com/emqx/emqx/pull/10378) Implemented Pulsar Producer Bridge and only producer role is supported now. +- [#10075](https://github.com/emqx/emqx/pull/10075) Add node rebalance/node evacuation functionality. + See also: [design doc](https://github.com/emqx/eip/blob/main/active/0020-node-rebalance.md) -- [#10408](https://github.com/emqx/emqx/pull/10408) Introduced 3 built-in functions in the rule engine SQL-like language for creating values of the MongoDB date type. +- [#10378](https://github.com/emqx/emqx/pull/10378) Implement Pulsar Producer Bridge, which supports publishing messages to Pulsar from MQTT topics. -- [#10409](https://github.com/emqx/emqx/pull/10409) [#10337](#10337) Supported [Protocol Buffers](https://protobuf.dev/) and [Apache Avro](https://avro.apache.org/) schemas in Schema Registry. +- [#10408](https://github.com/emqx/emqx/pull/10408) The rule engine SQL-like language has got three more built-in functions for creating values of the MongoDB date type. These functions are useful for rules with MongoDB bridge actions only and not supported in other actions. -- [#10425](https://github.com/emqx/emqx/pull/10425) Implemented OpenTSDB data bridge. +- [#10409](https://github.com/emqx/emqx/pull/10409) Add support for [Protocol Buffers](https://protobuf.dev/) schemas in Schema Registry. -- [#10498](https://github.com/emqx/emqx/pull/10498) Implemented Oracle Database Bridge. +- [#10425](https://github.com/emqx/emqx/pull/10425) Implement OpenTSDB data bridge. -- [#10560](https://github.com/emqx/emqx/pull/10560) Added enterprise data bridge for Apache IoTDB. +- [#10498](https://github.com/emqx/emqx/pull/10498) Implement Oracle Database Bridge, which supports publishing messages to Oracle Database from MQTT topics. -- [#10417](https://github.com/emqx/emqx/pull/10417) Improved get config items performance by eliminating temporary references. +- [#10560](https://github.com/emqx/emqx/pull/10560) Add enterprise data bridge for Apache IoTDB. -- [#10430](https://github.com/emqx/emqx/pull/10430) Simplified the configuration of the `retainer` feature. Marked `flow_control` as a non-importance field. +- [#10648](https://github.com/emqx/emqx/pull/10648) Refactor the directory structure of the RocketMQ data bridge. -- [#10511](https://github.com/emqx/emqx/pull/10511) Improved the security and privacy of some resource logs by masking sensitive information in the log. +- [#10650](https://github.com/emqx/emqx/pull/10650) Refactor the directory structure of the TDEngine data bridge. -- [#10525](https://github.com/emqx/emqx/pull/10525) Reduced resource usage per MQTT packet handling. +- [#10662](https://github.com/emqx/emqx/pull/10662) Refactor the directory structure of the PostgreSQL && Matrix && Timescale data bridges. -- [#10528](https://github.com/emqx/emqx/pull/10528) Reduced memory footprint in hot code path. The hot path includes the code that is frequently executed in core functionalities such as message handling, connection management, authentication, and authorization. +- [#10679](https://github.com/emqx/emqx/pull/10679) Refactor the directory structure of the InfluxDB data bridge. -- [#10591](https://github.com/emqx/emqx/pull/10591) [#10625](https://github.com/emqx/emqx/pull/10625) Improved the configuration of the limiter. +- [#9927](https://github.com/emqx/emqx/pull/9927) Introduce support for the File Transfer over MQTT feature as described in [EIP-0021](https://github.com/emqx/eip), with support to publish transferred files either to the node-local file system or to the S3 API compatible remote object storage. - - Reduced the complexity of the limiter's configuration. +- [#10376](https://github.com/emqx/emqx/pull/10376) Simplify the configuration of the limiter feature and optimize some codes + - Rename `message_in` to `messages` + - Rename `bytes_in` to `bytes` + - Use `burst` instead of `capacity` + - Hide non-importance fields + - Optimize limiter instances in different rate settings - - Updated the `configs/limiter` API to suit this refactor. +- [#10417](https://github.com/emqx/emqx/pull/10417) Improve get config performance by eliminating temporary references. - - Reduced the memory usage of the limiter configuration. +- [#10430](https://github.com/emqx/emqx/pull/10430) Simplify the configuration of the `retainer` feature. + - Mark `flow_control` as non-importance field. -- [#10487](https://github.com/emqx/emqx/pull/10487) Optimized the instance of limiter for whose rate is `infinity` to reduce memory and CPU usage. +- [#10487](https://github.com/emqx/emqx/pull/10487) Optimize the instance of limiter for whose rate is `infinity` to reduce memory and CPU usage. -- [#10490](https://github.com/emqx/emqx/pull/10490) Removed the default limit of connect rate which used to be `1000/s`. +- [#10490](https://github.com/emqx/emqx/pull/10490) Remove the default limit of connect rate which used to be `1000/s` -- [#10077](https://github.com/emqx/emqx/pull/10077) Added support for QUIC TLS password-protected certificate file. +- [#10511](https://github.com/emqx/emqx/pull/10511) Improve the security and privacy of some resource logs by masking sensitive information in the data. + +- [#10525](https://github.com/emqx/emqx/pull/10525) Reduce resource usage per MQTT packet handling. + + +- [#10528](https://github.com/emqx/emqx/pull/10528) Reduce memory footprint in hot code path. + +- [#10591](https://github.com/emqx/emqx/pull/10591) Improve the configuration of the limiter. + - Simplify the memory representation of the limiter configuration. + - Make sure the node-level limiter can really work when the listener's limiter configuration is omitted. + +- [#10625](https://github.com/emqx/emqx/pull/10625) Simplify limiter configuration. + - Reduce the complexity of the limiter's configuration. + e.g. now users can use `limiter.messages_rate = 1000/s` to quickly set the node-level limit for the message publish. + - Update the `configs/limiter` API to suit this refactor. ## Bug Fixes -- [#10340](https://github.com/emqx/emqx/pull/10340) Fixed the issue that could lead to crash logs being printed when stopping EMQX via `systemd`. +- [#10340](https://github.com/emqx/emqx/pull/10340) Fixed the issue that could lead to crash logs being printed when stopping EMQX via systemd. + ``` + 2023-03-29T16:43:25.915761+08:00 [error] Generic server memsup terminating. Reason: {port_died,normal}. Last message: {'EXIT',<0.2117.0>,{port_died,normal}}. State: [{data,[{"Timeout",60000}]},{items,{"Memory Usage",[{"Allocated",929959936},{"Total",3832242176}]}},{items,{"Worst Memory User",[{"Pid",<0.2031.0>},{"Memory",4720472}]}}]. + 2023-03-29T16:43:25.924764+08:00 [error] crasher: initial call: memsup:init/1, pid: <0.2116.0>, registered_name: memsup, exit: {{port_died,normal},[{gen_server,handle_common_reply,8,[{file,"gen_server.erl"},{line,811}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}]}, ancestors: [os_mon_sup,<0.2114.0>], message_queue_len: 0, messages: [], links: [<0.2115.0>], dictionary: [], trap_exit: true, status: running, heap_size: 4185, stack_size: 29, reductions: 187637; neighbours: + 2023-03-29T16:43:25.924979+08:00 [error] Supervisor: {local,os_mon_sup}. Context: child_terminated. Reason: {port_died,normal}. Offender: id=memsup,pid=<0.2116.0>. + ``` - `2023-03-29T16:43:25.915761+08:00 [error] Generic server memsup terminating. Reason: {port_died,normal}. Last message: {'EXIT',<0.2117.0>,{port_died,normal}}. State: [{data,[{"Timeout",60000}]},{items,{"Memory Usage",[{"Allocated",929959936},{"Total",3832242176}]}},{items,{"Worst Memory User",[{"Pid",<0.2031.0>},{"Memory",4720472}]}}]. 2023-03-29T16:43:25.924764+08:00 [error] crasher: initial call: memsup:init/1, pid: <0.2116.0>, registered_name: memsup, exit: {{port_died,normal},[{gen_server,handle_common_reply,8,[{file,"gen_server.erl"},{line,811}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}]}, ancestors: [os_mon_sup,<0.2114.0>], message_queue_len: 0, messages: [], links: [<0.2115.0>], dictionary: [], trap_exit: true, status: running, heap_size: 4185, stack_size: 29, reductions: 187637; neighbours: 2023-03-29T16:43:25.924979+08:00 [error] Supervisor: {local,os_mon_sup}. Context: child_terminated. Reason: {port_died,normal}. Offender: id=memsup,pid=<0.2116.0>.` +- [#10369](https://github.com/emqx/emqx/pull/10369) Fix error in `/api/v5/monitor_current` API endpoint that happens when some EMQX nodes are down. -- [#10369](https://github.com/emqx/emqx/pull/10369) Fixed error in `/api/v5/monitor_current` API endpoint that happens when some EMQX nodes are down. + Prior to this fix, sometimes the request returned HTTP code 500 and the following message: + ``` + {"code":"INTERNAL_ERROR","message":"error, badarg, [{erlang,'++',[{error,nodedown},[{node,'emqx@10.42.0.150'}]], ... + ``` - Prior to this fix, sometimes the request returned HTTP code 500 and the following message: +- [#10407](https://github.com/emqx/emqx/pull/10407) Improve 'emqx_alarm' performance by using Mnesia dirty operations and avoiding + unnecessary calls from 'emqx_resource_manager' to reactivate alarms that have been already activated. + Use new safe 'emqx_alarm' API to activate/deactivate alarms to ensure that emqx_resource_manager + doesn't crash because of alarm timeouts. + The crashes were possible when the following conditions co-occurred: + - a relatively high number of failing resources, e.g. bridges tried to activate alarms on re-occurring errors; + - the system experienced a very high load. - `{"code":"INTERNAL_ERROR","message":"error, badarg, [{erlang,'++',[{error,nodedown},[{node,'emqx@10.42.0.150'}]], ...` +- [#10420](https://github.com/emqx/emqx/pull/10420) Fix HTTP path handling when composing the URL for the HTTP requests in authentication and authorization modules. + * Avoid unnecessary URL normalization since we cannot assume that external servers treat original and normalized URLs equally. This led to bugs like [#10411](https://github.com/emqx/emqx/issues/10411). + * Fix the issue that path segments could be HTTP encoded twice. -- [#10407](https://github.com/emqx/emqx/pull/10407) Fixed the crash issue of the alarm system. +- [#10422](https://github.com/emqx/emqx/pull/10422) Fixed a bug where external plugins could not be configured via environment variables in a lone-node cluster. - - Leverage Mnesia dirty operations and circumvent extraneous calls to enhance 'emqx_alarm' performance. +- [#10448](https://github.com/emqx/emqx/pull/10448) Fix a compatibility issue of limiter configuration introduced by v5.0.23 which broke the upgrade from previous versions if the `capacity` is `infinity`. - - Use 'emqx_resource_manager' for reactivating alarms that have already been triggered. + In v5.0.23 we have replaced `capacity` with `burst`. After this fix, a `capacity = infinity` config will be automatically converted to equivalent `burst = 0`. - - Implement the newly developed, fail-safe 'emqx_alarm' API to control the activation and deactivation of alarms, thus preventing 'emqx_resource_manager' from crashing due to alarm timeouts. +- [#10462](https://github.com/emqx/emqx/pull/10462) Deprecate config `broker.shared_dispatch_ack_enabled`. + This was designed to avoid dispatching messages to a shared-subscription session which has the client disconnected. + However since v5.0.9, this feature is no longer useful because the shared-subscrption messages in a expired session will be redispatched to other sessions in the group. + See also: https://github.com/emqx/emqx/pull/9104 - - The alarm system is susceptible to crashing under these concurrent conditions: +- [#10463](https://github.com/emqx/emqx/pull/10463) Improve bridges API error handling. + If Webhook bridge URL is not valid, bridges API will return '400' error instead of '500'. - - A significant number of resources fail, such as when bridges continuously attempt to trigger alarms due to recurring errors. +- [#10484](https://github.com/emqx/emqx/pull/10484) Fix the issue that the priority of the configuration cannot be set during rolling upgrade. + For example, when authorization is modified in v5.0.21 and then upgraded v5.0.23 through rolling upgrade, + the authorization will be restored to the default. - - The system is under an extremely high load. +- [#10495](https://github.com/emqx/emqx/pull/10495) Add the limiter API `/configs/limiter` which was deleted by mistake back. -- [#10420](https://github.com/emqx/emqx/pull/10420) Fixed HTTP path handling when composing the URL for the HTTP requests in authentication and authorization modules. +- [#10500](https://github.com/emqx/emqx/pull/10500) Add several fixes, enhancements and features in Mria: + - protect `mria:join/1,2` with a global lock to prevent conflicts between + two nodes trying to join each other simultaneously + [Mria PR](https://github.com/emqx/mria/pull/137) + - implement new function `mria:sync_transaction/4,3,2`, which blocks the caller until + a transaction is imported to the local node (if the local node is a replicant, otherwise, + it behaves exactly the same as `mria:transaction/3,2`) + [Mria PR](https://github.com/emqx/mria/pull/136) + - optimize `mria:running_nodes/0` + [Mria PR](https://github.com/emqx/mria/pull/135) + - optimize `mria:ro_transaction/2` when called on a replicant node + [Mria PR](https://github.com/emqx/mria/pull/134). - - Avoid unnecessary URL normalization since we cannot assume that external servers treat original and normalized URLs equally. This led to bugs like [#10411](https://github.com/emqx/emqx/issues/10411). +- [#10518](https://github.com/emqx/emqx/pull/10518) Add the following fixes and features in Mria: + - call `mria_rlog:role/1` safely in mria_membership to ensure that mria_membership + gen_server won't crash if RPC to another node fails + [Mria PR](https://github.com/emqx/mria/pull/139) + - Add extra field to ?rlog_sync table to facilitate extending this functionality in future + [Mria PR](https://github.com/emqx/mria/pull/138). - - Fixed the issue that path segments could be HTTP encoded twice. +- [#10556](https://github.com/emqx/emqx/pull/10556) Wrap potentially sensitive data in `emqx_connector_http` if `Authorization` headers are being passed at initialization. -- [#10422](https://github.com/emqx/emqx/pull/10422) Fixed a bug where external plugins could not be configured via environment variables in a lone-node cluster. +- [#10659](https://github.com/emqx/emqx/pull/10659) Fix the issue where emqx cannot start when `sysmon.os.mem_check_interval` is disabled. -- [#10448](https://github.com/emqx/emqx/pull/10448) Fixed a compatibility issue of limiter configuration introduced by e5.0.3 which broke the upgrade from previous versions if the `capacity` is `infinity`. +- [#10717](https://github.com/emqx/emqx/pull/10717) Fixed an issue where the buffering layer processes could use a lot of CPU when inflight window is full. - In e5.0.3 we have replaced `capacity` with `burst`. After this fix, a `capacity = infinity` config will be automatically converted to equivalent `burst = 0`. +- [#10724](https://github.com/emqx/emqx/pull/10724) A summary has been added for all endpoints in the HTTP API documentation (accessible at "http://emqx_host_name:18083/api-docs"). -- [#10462](https://github.com/emqx/emqx/pull/10462) Deprecated config `broker.shared_dispatch_ack_enabled`. This was designed to avoid dispatching messages to a shared-subscription session that has the client disconnected. However, since e5.0.0, this feature is no longer helpful because the shared-subscription messages in an expired session will be redispatched to other sessions in the group. See also: . +- [#10726](https://github.com/emqx/emqx/pull/10726) Validate Health Check Interval and Auto Restart Interval against the range from 1ms to 1 hour. -- [#10463](https://github.com/emqx/emqx/pull/10463) Improved bridges API error handling. If Webhook bridge URL is not valid, the bridges API will return '400' error instead of '500'. +- [#10728](https://github.com/emqx/emqx/pull/10728) Fixed an issue where the rule engine was unable to access variables exported by `FOREACH` in the `DO` clause. -- [#10484](https://github.com/emqx/emqx/pull/10484) Fixed the issue that the priority of the configuration cannot be set during the rolling upgrade. For example, when authorization is modified in e5.0.2 and then upgraded e5.0.3 through the rolling upgrade, the authorization will be restored to the default. + Given a payload: `{"date": "2023-05-06", "array": ["a"]}`, as well as the following SQL statement: + ``` + FOREACH payload.date as date, payload.array as elem + DO date, elem + FROM "t/#" + ``` + Prior to the fix, the `date` variable exported by `FOREACH` could not be accessed in the `DO` clause of the above SQL, resulting in the following output for the SQL statement: + `[{"elem": "a","date": "undefined"}]`. + After the fix, the output of the SQL statement is: `[{"elem": "a","date": "2023-05-06"}]` -- [#10495](https://github.com/emqx/emqx/pull/10495) Added the limiter API `/configs/limiter` which was deleted by mistake back. +- [#10742](https://github.com/emqx/emqx/pull/10742) Check the correctness of the rules before saving the authorization file source. + Previously, Saving wrong rules could lead to restart failure. -- [#10500](https://github.com/emqx/emqx/pull/10500) Added several fixes, enhancements, and features in Mria: +- [#10743](https://github.com/emqx/emqx/pull/10743) Fixes an issue where trying to get a bridge info or metrics could result in a crash when a node is joining a cluster. - - Protect `mria:join/1,2` with a global lock to prevent conflicts between two nodes trying to join each other simultaneously [Mria PR](https://github.com/emqx/mria/pull/137) +- [#10746](https://github.com/emqx/emqx/pull/10746) Add missing support of the event `$events/delivery_dropped` into the rule engine test API `rule_test`. - - Implement new function `mria:sync_transaction/4,3,2`, which blocks the caller until a transaction is imported to the local node (if the local node is a replicant, otherwise, it behaves exactly the same as `mria:transaction/3,2`) [Mria PR](https://github.com/emqx/mria/pull/136) +- [#10747](https://github.com/emqx/emqx/pull/10747) Refactor date and time functions, `format_date` and `date_to_unix_ts`, in the rule engine to fix the implementation problem. - - Optimize `mria:running_nodes/0` [Mria PR](https://github.com/emqx/mria/pull/135) +- [#10755](https://github.com/emqx/emqx/pull/10755) Fixed data bridge resource update race condition. - - Optimize `mria:ro_transaction/2` when called on a replicant node [Mria PR](https://github.com/emqx/mria/pull/134). + In the 'delete + create' process for EMQX resource updates, + long bridge creation times could cause dashboard request timeouts. + If a bridge resource update was initiated before completion of its creation, + it led to an erroneous deletion from the runtime, despite being present in the config file. -- [#10518](https://github.com/emqx/emqx/pull/10518) Added the following fixes and features in Mria: + This fix addresses the race condition in bridge resource updates, + ensuring the accurate identification and addition of new resources, + maintaining consistency between runtime and configuration file statuses. - - Call `mria_rlog:role/1` safely in mria_membership to ensure that mria_membership gen_server won't crash if RPC to another node fails [Mria PR](https://github.com/emqx/mria/pull/139) +- [#10760](https://github.com/emqx/emqx/pull/10760) Fix Internal Error 500 that occurred sometimes when bridge statistics page was updated while a node was (re)joining the cluster. - - Add an extra field to `?rlog_sync` table to facilitate extending this functionality in future [Mria PR](https://github.com/emqx/mria/pull/138). +- [#10761](https://github.com/emqx/emqx/pull/10761) Fixing the issue where the default value of SSL certificate for Dashboard Listener was not correctly interpolated, which caused HTTPS to be inaccessible when verify_peer and cacertfile were using the default configuration. -- [#10556](https://github.com/emqx/emqx/pull/10556) Wrapped potentially sensitive data in `emqx_connector_http` if `Authorization` headers are being passed at initialization. +- [#10801](https://github.com/emqx/emqx/pull/10801) Avoid duplicated percent decode the topic name in API `/topics/{topic}` and `/topics`. -- [#10571](https://github.com/emqx/emqx/pull/10571) Stopped emitting useless crash report when EMQX stops. +- [#10817](https://github.com/emqx/emqx/pull/10817) Fix the error of not being able to configure `auto_restart_interval` as infinity -- [#10659](https://github.com/emqx/emqx/pull/10659) Fixed the issue where EMQX cannot start when `sysmon.os.mem_check_interval` is disabled. +- [#10672](https://github.com/emqx/emqx/pull/10672) Fix the issue where the lack of a default value for ssl_options in listeners results in startup failure. + For example, such command(`EMQX_LISTENERS__WSS__DEFAULT__BIND='0.0.0.0:8089' ./bin/emqx console`) would have caused a crash before. -- [#10717](https://github.com/emqx/emqx/pull/10717) Fixed an issue where the buffering layer processes could use a lot of CPU when inflight window is full. +- [#10738](https://github.com/emqx/emqx/pull/10738) Add support for the `Supertable` and `Create Tables Automatically` features of TDEngine to its data bridge. + Before this fix, an insert with a supertable in the template will fail, like this: + `insert into ${clientid} using msg TAGS (${clientid}) values (${ts},${msg})`. -- [#10724](https://github.com/emqx/emqx/pull/10724) A summary has been added for all endpoints in the HTTP API documentation (accessible at "http://:18083/api-docs"). - -- [#10726](https://github.com/emqx/emqx/pull/10726) Health Check Interval and Auto Restart Interval now support the range from 1ms to 1 hour. - -- [#10728](https://github.com/emqx/emqx/pull/10728) Fixed an issue where the rule engine was unable to access variables exported by `FOREACH` - `DO` clause. - - Given a payload: `{"date": "2023-05-06", "array": ["a"]}`, as well as the following SQL statement: - - `FOREACH payload.date as date, payload.array as elem DO date, elem FROM "t/#" -- {"date": "2023-05-06", "array": ["a"]}` - - Prior to the fix, the `date` variable exported by `FOREACH` could not be accessed in the `DO` clause of the above SQL, resulting in the following output for the SQL statement: `[{"elem": "a","date": "undefined"}]`. - -- [#10742](https://github.com/emqx/emqx/pull/10742) Correctness check of the rules is enforced before saving the authorization file source. Previously, Saving wrong rules could lead to EMQX restart failure. - -- [#10743](https://github.com/emqx/emqx/pull/10743) Fixed an issue where trying to get bridge info or metrics could result in a crash when a node is joining a cluster. - -- [#10755](https://github.com/emqx/emqx/pull/10755) Fixed data bridge resource update race condition. - - In the 'delete + create' process for EMQX resource updates, long bridge creation times could cause dashboard request timeouts. If a bridge resource update was initiated before completion of its creation, it led to an erroneous deletion from the runtime, despite being present in the config file. - - This fix addresses the race condition in bridge resource updates, ensuring the accurate identification and addition of new resources, and maintaining consistency between runtime and configuration file statuses. - -- [#10761](https://github.com/emqx/emqx/pull/10761) Fixed the issue where the default value of SSL certificate for Dashboard Listener was not correctly interpolated, which caused HTTPS to be inaccessible when `verify_peer` and `cacertfile` were using the default configuration. - -- [#10672](https://github.com/emqx/emqx/pull/10672) Fixed the issue where the lack of a default value for `ssl_options` in listeners results in startup failure. For example, such command(`EMQX_LISTENERS__WSS__DEFAULT__BIND='0.0.0.0:8089' ./bin/emqx console`) would have caused a crash before. - -- [#10738](https://github.com/emqx/emqx/pull/10738) TDEngine data bridge now supports "Supertable" and "Create Tables Automatically". Before this fix, an insert with a supertable in the template will fail, like this: - - - `insert into ${clientid} using msg TAGS (${clientid}) values (${ts},${msg})`. - -- [#10746](https://github.com/emqx/emqx/pull/10746) Add missing support of the event `$events/delivery_dropped` into the rule engine test API `rule_test`. - -- [#10747](https://github.com/emqx/emqx/pull/10747) Ported some time formating fixes in Rule-Engine functions from version 4.4. - -- [#10760](https://github.com/emqx/emqx/pull/10760) Fix "internal error 500" when getting bridge statistics page while a node is joining the cluster. - -- [#10801](https://github.com/emqx/emqx/pull/10801) Avoid double percent-decode for topic name in API `/topics/{topic}` and `/topics`. - -- [#10817](https://github.com/emqx/emqx/pull/10817) Fix a config value handling for bridge resource option `auto_restart_interval`, now it can be set to `infinity`. +- [#10741](https://github.com/emqx/emqx/pull/10741) Fix password leaking on stacktrace for Oracle Database. From eaba36fdca7fbd0245c70f8489c7823647032283 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 26 May 2023 15:43:01 +0200 Subject: [PATCH 18/20] docs: restore e5.0.4 changelog to manually updated version --- changes/e5.0.4.en.md | 259 ++++++++++++++++++------------------------- 1 file changed, 105 insertions(+), 154 deletions(-) diff --git a/changes/e5.0.4.en.md b/changes/e5.0.4.en.md index 3f5c91f92..40b77da1b 100644 --- a/changes/e5.0.4.en.md +++ b/changes/e5.0.4.en.md @@ -2,225 +2,176 @@ ## Enhancements -- [#10389](https://github.com/emqx/emqx/pull/10389) Unify the config formats for `cluster.core_nodes` and `cluster.statics.seeds`. - Now they both support formats in array `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` or semicolon-separated string `"emqx1@127.0.0.1,emqx2@127.0.0.1"`. +- [#10389](https://github.com/emqx/emqx/pull/10389) Unified the configuration formats for `cluster.core_nodes` and `cluster.statics.seeds`. Now they both support formats in array `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` and the comma-separated string `"emqx1@127.0.0.1,emqx2@127.0.0.1"`. -- [#10392](https://github.com/emqx/emqx/pull/10392) A new function to convert a formatted date to an integer timestamp has been added: date_to_unix_ts/3 +- [#10392](https://github.com/emqx/emqx/pull/10392) Introduced a new function to convert a formatted date to an integer timestamp: date_to_unix_ts/3. -- [#10426](https://github.com/emqx/emqx/pull/10426) Optimize the configuration priority mechanism to fix the issue where the configuration - changes made to `etc/emqx.conf` do not take effect after restarting EMQX. + `date_to_unix_ts(TimeUnit, FormatString, InputDateTimeString)` - More introduction about the new mechanism: [Configure Override Rules](https://www.emqx.io/docs/en/v5.0/configuration/configuration.html#configure-override-rules) +- [#10426](https://github.com/emqx/emqx/pull/10426) Optimized the configuration priority mechanism to fix the issue where the configuration changes made to `etc/emqx.conf` do not take effect after restarting EMQX. -- [#10457](https://github.com/emqx/emqx/pull/10457) Deprecates the integration with StatsD. + More information about the new mechanism: [Configure Override Rules](https://www.emqx.io/docs/en/v5.0/configuration/configuration.html#configure-override-rules) - There seemd to be no user using StatsD integration, so we have decided to hide this feature - for now. We will either remove or revive it based on requirements in the future. +- [#10457](https://github.com/emqx/emqx/pull/10457) Deprecated the integration with StatsD. -- [#10458](https://github.com/emqx/emqx/pull/10458) Set the level of plugin configuration options to low level, - in most cases, users only need to manage plugins on the dashboard - without the need for manual modification, so we lowered the level. +- [#10458](https://github.com/emqx/emqx/pull/10458) Set the level of plugin configuration options to low, users usually manage the plugins through the dashboard, rarely modify them manually, so we lowered the level. -- [#10491](https://github.com/emqx/emqx/pull/10491) Rename `etcd.ssl` to `etcd.ssl_options` to keep all of SSL options consistent in the configuration file. +- [#10491](https://github.com/emqx/emqx/pull/10491) Renamed `etcd.ssl` to `etcd.ssl_options` to keep all SSL options consistent in the configuration file. -- [#10512](https://github.com/emqx/emqx/pull/10512) Improved the storage format of Unicode characters in data files, - Now we can store Unicode characters normally. - For example: "SELECT * FROM \"t/1\" WHERE clientid = \"-测试专用-\"" +- [#10512](https://github.com/emqx/emqx/pull/10512) Improved the storage format of Unicode characters in data files, Now we can store Unicode characters. For example: `SELECT * FROM "t/1" WHERE clientid = "-测试专用-"`. -- [#10568](https://github.com/emqx/emqx/pull/10568) Add shutdown counter information to `emqx ctl listeners` command +- [#10568](https://github.com/emqx/emqx/pull/10568) Added `shutdown_counter` printout to `emqx ctl listeners` command. -- [#10571](https://github.com/emqx/emqx/pull/10571) Do not emit useless crash report when EMQX stops. - Previously, when EMQX (and `emqx_topic_metrics` in particular) stopped and removed underlying tables, some messages were still being handled and crashed. +- [#10588](https://github.com/emqx/emqx/pull/10588) Increased the time precision of trace logs from second to microsecond. For example, change from `2023-05-02T08:43:50+00:00` to `2023-05-02T08:43:50.237945+00:00`. -- [#10588](https://github.com/emqx/emqx/pull/10588) Increase the time precision of trace logs from second to microsecond. - For example, change from `2023-05-02T08:43:50+00:00` to `2023-05-02T08:43:50.237945+00:00`. +- [#10623](https://github.com/emqx/emqx/pull/10623) Renamed `max_message_queue_len` to `max_mailbox_size` in the `force_shutdown` configuration. The old name is kept as an alias, so this change is backward compatible. -- [#10623](https://github.com/emqx/emqx/pull/10623) Renamed `max_message_queue_len` to `max_mailbox_size` in the `force_shutdown` configuration. Old name is kept as an alias, so this change is backward compatible. +- [#10713](https://github.com/emqx/emqx/pull/10713) Hide the `resource_option.request_timeout` of the webhook and it will use the value of `http` `request_timeout`. -- [#10713](https://github.com/emqx/emqx/pull/10713) We hide the request_timeout in resource_option of the webhook to keep it consistent with the http request_timeout of the webhook. - From now on, when configuring a webhook through API or configuration files, - it is no longer necessary to configure the request_timeout of the resource. Only configuring the http request_timeout is sufficient, and the request_timeout in the resource will automatically be consistent with the http request_timeout. +- [#10075](https://github.com/emqx/emqx/pull/10075) Added node rebalance/node evacuation functionality. See also: [EIP doc](https://github.com/emqx/eip/blob/main/active/0020-node-rebalance.md) -- [#10075](https://github.com/emqx/emqx/pull/10075) Add node rebalance/node evacuation functionality. - See also: [design doc](https://github.com/emqx/eip/blob/main/active/0020-node-rebalance.md) +- [#10378](https://github.com/emqx/emqx/pull/10378) Implemented Pulsar Producer Bridge and only producer role is supported now. -- [#10378](https://github.com/emqx/emqx/pull/10378) Implement Pulsar Producer Bridge, which supports publishing messages to Pulsar from MQTT topics. +- [#10408](https://github.com/emqx/emqx/pull/10408) Introduced 3 built-in functions in the rule engine SQL-like language for creating values of the MongoDB date type. -- [#10408](https://github.com/emqx/emqx/pull/10408) The rule engine SQL-like language has got three more built-in functions for creating values of the MongoDB date type. These functions are useful for rules with MongoDB bridge actions only and not supported in other actions. +- [#10409](https://github.com/emqx/emqx/pull/10409) [#10337](#10337) Supported [Protocol Buffers](https://protobuf.dev/) and [Apache Avro](https://avro.apache.org/) schemas in Schema Registry. -- [#10409](https://github.com/emqx/emqx/pull/10409) Add support for [Protocol Buffers](https://protobuf.dev/) schemas in Schema Registry. +- [#10425](https://github.com/emqx/emqx/pull/10425) Implemented OpenTSDB data bridge. -- [#10425](https://github.com/emqx/emqx/pull/10425) Implement OpenTSDB data bridge. +- [#10498](https://github.com/emqx/emqx/pull/10498) Implemented Oracle Database Bridge. -- [#10498](https://github.com/emqx/emqx/pull/10498) Implement Oracle Database Bridge, which supports publishing messages to Oracle Database from MQTT topics. +- [#10560](https://github.com/emqx/emqx/pull/10560) Added enterprise data bridge for Apache IoTDB. -- [#10560](https://github.com/emqx/emqx/pull/10560) Add enterprise data bridge for Apache IoTDB. +- [#10417](https://github.com/emqx/emqx/pull/10417) Improved get config items performance by eliminating temporary references. -- [#10648](https://github.com/emqx/emqx/pull/10648) Refactor the directory structure of the RocketMQ data bridge. +- [#10430](https://github.com/emqx/emqx/pull/10430) Simplified the configuration of the `retainer` feature. Marked `flow_control` as a non-importance field. -- [#10650](https://github.com/emqx/emqx/pull/10650) Refactor the directory structure of the TDEngine data bridge. +- [#10511](https://github.com/emqx/emqx/pull/10511) Improved the security and privacy of some resource logs by masking sensitive information in the log. -- [#10662](https://github.com/emqx/emqx/pull/10662) Refactor the directory structure of the PostgreSQL && Matrix && Timescale data bridges. +- [#10525](https://github.com/emqx/emqx/pull/10525) Reduced resource usage per MQTT packet handling. -- [#10679](https://github.com/emqx/emqx/pull/10679) Refactor the directory structure of the InfluxDB data bridge. +- [#10528](https://github.com/emqx/emqx/pull/10528) Reduced memory footprint in hot code path. The hot path includes the code that is frequently executed in core functionalities such as message handling, connection management, authentication, and authorization. -- [#9927](https://github.com/emqx/emqx/pull/9927) Introduce support for the File Transfer over MQTT feature as described in [EIP-0021](https://github.com/emqx/eip), with support to publish transferred files either to the node-local file system or to the S3 API compatible remote object storage. +- [#10591](https://github.com/emqx/emqx/pull/10591) [#10625](https://github.com/emqx/emqx/pull/10625) Improved the configuration of the limiter. -- [#10376](https://github.com/emqx/emqx/pull/10376) Simplify the configuration of the limiter feature and optimize some codes - - Rename `message_in` to `messages` - - Rename `bytes_in` to `bytes` - - Use `burst` instead of `capacity` - - Hide non-importance fields - - Optimize limiter instances in different rate settings + - Reduced the complexity of the limiter's configuration. -- [#10417](https://github.com/emqx/emqx/pull/10417) Improve get config performance by eliminating temporary references. + - Updated the `configs/limiter` API to suit this refactor. -- [#10430](https://github.com/emqx/emqx/pull/10430) Simplify the configuration of the `retainer` feature. - - Mark `flow_control` as non-importance field. + - Reduced the memory usage of the limiter configuration. -- [#10487](https://github.com/emqx/emqx/pull/10487) Optimize the instance of limiter for whose rate is `infinity` to reduce memory and CPU usage. +- [#10487](https://github.com/emqx/emqx/pull/10487) Optimized the instance of limiter for whose rate is `infinity` to reduce memory and CPU usage. -- [#10490](https://github.com/emqx/emqx/pull/10490) Remove the default limit of connect rate which used to be `1000/s` +- [#10490](https://github.com/emqx/emqx/pull/10490) Removed the default limit of connect rate which used to be `1000/s`. -- [#10511](https://github.com/emqx/emqx/pull/10511) Improve the security and privacy of some resource logs by masking sensitive information in the data. - -- [#10525](https://github.com/emqx/emqx/pull/10525) Reduce resource usage per MQTT packet handling. - - -- [#10528](https://github.com/emqx/emqx/pull/10528) Reduce memory footprint in hot code path. - -- [#10591](https://github.com/emqx/emqx/pull/10591) Improve the configuration of the limiter. - - Simplify the memory representation of the limiter configuration. - - Make sure the node-level limiter can really work when the listener's limiter configuration is omitted. - -- [#10625](https://github.com/emqx/emqx/pull/10625) Simplify limiter configuration. - - Reduce the complexity of the limiter's configuration. - e.g. now users can use `limiter.messages_rate = 1000/s` to quickly set the node-level limit for the message publish. - - Update the `configs/limiter` API to suit this refactor. +- [#10077](https://github.com/emqx/emqx/pull/10077) Added support for QUIC TLS password-protected certificate file. ## Bug Fixes -- [#10340](https://github.com/emqx/emqx/pull/10340) Fixed the issue that could lead to crash logs being printed when stopping EMQX via systemd. - ``` - 2023-03-29T16:43:25.915761+08:00 [error] Generic server memsup terminating. Reason: {port_died,normal}. Last message: {'EXIT',<0.2117.0>,{port_died,normal}}. State: [{data,[{"Timeout",60000}]},{items,{"Memory Usage",[{"Allocated",929959936},{"Total",3832242176}]}},{items,{"Worst Memory User",[{"Pid",<0.2031.0>},{"Memory",4720472}]}}]. - 2023-03-29T16:43:25.924764+08:00 [error] crasher: initial call: memsup:init/1, pid: <0.2116.0>, registered_name: memsup, exit: {{port_died,normal},[{gen_server,handle_common_reply,8,[{file,"gen_server.erl"},{line,811}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}]}, ancestors: [os_mon_sup,<0.2114.0>], message_queue_len: 0, messages: [], links: [<0.2115.0>], dictionary: [], trap_exit: true, status: running, heap_size: 4185, stack_size: 29, reductions: 187637; neighbours: - 2023-03-29T16:43:25.924979+08:00 [error] Supervisor: {local,os_mon_sup}. Context: child_terminated. Reason: {port_died,normal}. Offender: id=memsup,pid=<0.2116.0>. - ``` +- [#10340](https://github.com/emqx/emqx/pull/10340) Fixed the issue that could lead to crash logs being printed when stopping EMQX via `systemd`. -- [#10369](https://github.com/emqx/emqx/pull/10369) Fix error in `/api/v5/monitor_current` API endpoint that happens when some EMQX nodes are down. + `2023-03-29T16:43:25.915761+08:00 [error] Generic server memsup terminating. Reason: {port_died,normal}. Last message: {'EXIT',<0.2117.0>,{port_died,normal}}. State: [{data,[{"Timeout",60000}]},{items,{"Memory Usage",[{"Allocated",929959936},{"Total",3832242176}]}},{items,{"Worst Memory User",[{"Pid",<0.2031.0>},{"Memory",4720472}]}}]. 2023-03-29T16:43:25.924764+08:00 [error] crasher: initial call: memsup:init/1, pid: <0.2116.0>, registered_name: memsup, exit: {{port_died,normal},[{gen_server,handle_common_reply,8,[{file,"gen_server.erl"},{line,811}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}]}, ancestors: [os_mon_sup,<0.2114.0>], message_queue_len: 0, messages: [], links: [<0.2115.0>], dictionary: [], trap_exit: true, status: running, heap_size: 4185, stack_size: 29, reductions: 187637; neighbours: 2023-03-29T16:43:25.924979+08:00 [error] Supervisor: {local,os_mon_sup}. Context: child_terminated. Reason: {port_died,normal}. Offender: id=memsup,pid=<0.2116.0>.` - Prior to this fix, sometimes the request returned HTTP code 500 and the following message: - ``` - {"code":"INTERNAL_ERROR","message":"error, badarg, [{erlang,'++',[{error,nodedown},[{node,'emqx@10.42.0.150'}]], ... - ``` +- [#10369](https://github.com/emqx/emqx/pull/10369) Fixed error in `/api/v5/monitor_current` API endpoint that happens when some EMQX nodes are down. -- [#10407](https://github.com/emqx/emqx/pull/10407) Improve 'emqx_alarm' performance by using Mnesia dirty operations and avoiding - unnecessary calls from 'emqx_resource_manager' to reactivate alarms that have been already activated. - Use new safe 'emqx_alarm' API to activate/deactivate alarms to ensure that emqx_resource_manager - doesn't crash because of alarm timeouts. - The crashes were possible when the following conditions co-occurred: - - a relatively high number of failing resources, e.g. bridges tried to activate alarms on re-occurring errors; - - the system experienced a very high load. + Prior to this fix, sometimes the request returned HTTP code 500 and the following message: -- [#10420](https://github.com/emqx/emqx/pull/10420) Fix HTTP path handling when composing the URL for the HTTP requests in authentication and authorization modules. - * Avoid unnecessary URL normalization since we cannot assume that external servers treat original and normalized URLs equally. This led to bugs like [#10411](https://github.com/emqx/emqx/issues/10411). - * Fix the issue that path segments could be HTTP encoded twice. + `{"code":"INTERNAL_ERROR","message":"error, badarg, [{erlang,'++',[{error,nodedown},[{node,'emqx@10.42.0.150'}]], ...` -- [#10422](https://github.com/emqx/emqx/pull/10422) Fixed a bug where external plugins could not be configured via environment variables in a lone-node cluster. +- [#10407](https://github.com/emqx/emqx/pull/10407) Fixed the crash issue of the alarm system. -- [#10448](https://github.com/emqx/emqx/pull/10448) Fix a compatibility issue of limiter configuration introduced by v5.0.23 which broke the upgrade from previous versions if the `capacity` is `infinity`. + - Leverage Mnesia dirty operations and circumvent extraneous calls to enhance 'emqx_alarm' performance. - In v5.0.23 we have replaced `capacity` with `burst`. After this fix, a `capacity = infinity` config will be automatically converted to equivalent `burst = 0`. + - Use 'emqx_resource_manager' for reactivating alarms that have already been triggered. -- [#10462](https://github.com/emqx/emqx/pull/10462) Deprecate config `broker.shared_dispatch_ack_enabled`. - This was designed to avoid dispatching messages to a shared-subscription session which has the client disconnected. - However since v5.0.9, this feature is no longer useful because the shared-subscrption messages in a expired session will be redispatched to other sessions in the group. - See also: https://github.com/emqx/emqx/pull/9104 + - Implement the newly developed, fail-safe 'emqx_alarm' API to control the activation and deactivation of alarms, thus preventing 'emqx_resource_manager' from crashing due to alarm timeouts. -- [#10463](https://github.com/emqx/emqx/pull/10463) Improve bridges API error handling. - If Webhook bridge URL is not valid, bridges API will return '400' error instead of '500'. + - The alarm system is susceptible to crashing under these concurrent conditions: -- [#10484](https://github.com/emqx/emqx/pull/10484) Fix the issue that the priority of the configuration cannot be set during rolling upgrade. - For example, when authorization is modified in v5.0.21 and then upgraded v5.0.23 through rolling upgrade, - the authorization will be restored to the default. + - A significant number of resources fail, such as when bridges continuously attempt to trigger alarms due to recurring errors. -- [#10495](https://github.com/emqx/emqx/pull/10495) Add the limiter API `/configs/limiter` which was deleted by mistake back. + - The system is under an extremely high load. -- [#10500](https://github.com/emqx/emqx/pull/10500) Add several fixes, enhancements and features in Mria: - - protect `mria:join/1,2` with a global lock to prevent conflicts between - two nodes trying to join each other simultaneously - [Mria PR](https://github.com/emqx/mria/pull/137) - - implement new function `mria:sync_transaction/4,3,2`, which blocks the caller until - a transaction is imported to the local node (if the local node is a replicant, otherwise, - it behaves exactly the same as `mria:transaction/3,2`) - [Mria PR](https://github.com/emqx/mria/pull/136) - - optimize `mria:running_nodes/0` - [Mria PR](https://github.com/emqx/mria/pull/135) - - optimize `mria:ro_transaction/2` when called on a replicant node - [Mria PR](https://github.com/emqx/mria/pull/134). +- [#10420](https://github.com/emqx/emqx/pull/10420) Fixed HTTP path handling when composing the URL for the HTTP requests in authentication and authorization modules. -- [#10518](https://github.com/emqx/emqx/pull/10518) Add the following fixes and features in Mria: - - call `mria_rlog:role/1` safely in mria_membership to ensure that mria_membership - gen_server won't crash if RPC to another node fails - [Mria PR](https://github.com/emqx/mria/pull/139) - - Add extra field to ?rlog_sync table to facilitate extending this functionality in future - [Mria PR](https://github.com/emqx/mria/pull/138). + - Avoid unnecessary URL normalization since we cannot assume that external servers treat original and normalized URLs equally. This led to bugs like [#10411](https://github.com/emqx/emqx/issues/10411). -- [#10556](https://github.com/emqx/emqx/pull/10556) Wrap potentially sensitive data in `emqx_connector_http` if `Authorization` headers are being passed at initialization. + - Fixed the issue that path segments could be HTTP encoded twice. -- [#10659](https://github.com/emqx/emqx/pull/10659) Fix the issue where emqx cannot start when `sysmon.os.mem_check_interval` is disabled. +- [#10422](https://github.com/emqx/emqx/pull/10422) Fixed a bug where external plugins could not be configured via environment variables in a lone-node cluster. -- [#10717](https://github.com/emqx/emqx/pull/10717) Fixed an issue where the buffering layer processes could use a lot of CPU when inflight window is full. +- [#10448](https://github.com/emqx/emqx/pull/10448) Fixed a compatibility issue of limiter configuration introduced by e5.0.3 which broke the upgrade from previous versions if the `capacity` is `infinity`. -- [#10724](https://github.com/emqx/emqx/pull/10724) A summary has been added for all endpoints in the HTTP API documentation (accessible at "http://emqx_host_name:18083/api-docs"). + In e5.0.3 we have replaced `capacity` with `burst`. After this fix, a `capacity = infinity` config will be automatically converted to equivalent `burst = 0`. -- [#10726](https://github.com/emqx/emqx/pull/10726) Validate Health Check Interval and Auto Restart Interval against the range from 1ms to 1 hour. +- [#10462](https://github.com/emqx/emqx/pull/10462) Deprecated config `broker.shared_dispatch_ack_enabled`. This was designed to avoid dispatching messages to a shared-subscription session that has the client disconnected. However, since e5.0.0, this feature is no longer helpful because the shared-subscription messages in an expired session will be redispatched to other sessions in the group. See also: . -- [#10728](https://github.com/emqx/emqx/pull/10728) Fixed an issue where the rule engine was unable to access variables exported by `FOREACH` in the `DO` clause. +- [#10463](https://github.com/emqx/emqx/pull/10463) Improved bridges API error handling. If Webhook bridge URL is not valid, the bridges API will return '400' error instead of '500'. - Given a payload: `{"date": "2023-05-06", "array": ["a"]}`, as well as the following SQL statement: - ``` - FOREACH payload.date as date, payload.array as elem - DO date, elem - FROM "t/#" - ``` - Prior to the fix, the `date` variable exported by `FOREACH` could not be accessed in the `DO` clause of the above SQL, resulting in the following output for the SQL statement: - `[{"elem": "a","date": "undefined"}]`. - After the fix, the output of the SQL statement is: `[{"elem": "a","date": "2023-05-06"}]` +- [#10484](https://github.com/emqx/emqx/pull/10484) Fixed the issue that the priority of the configuration cannot be set during the rolling upgrade. For example, when authorization is modified in e5.0.2 and then upgraded e5.0.3 through the rolling upgrade, the authorization will be restored to the default. -- [#10742](https://github.com/emqx/emqx/pull/10742) Check the correctness of the rules before saving the authorization file source. - Previously, Saving wrong rules could lead to restart failure. +- [#10495](https://github.com/emqx/emqx/pull/10495) Added the limiter API `/configs/limiter` which was deleted by mistake back. -- [#10743](https://github.com/emqx/emqx/pull/10743) Fixes an issue where trying to get a bridge info or metrics could result in a crash when a node is joining a cluster. +- [#10500](https://github.com/emqx/emqx/pull/10500) Added several fixes, enhancements, and features in Mria: -- [#10746](https://github.com/emqx/emqx/pull/10746) Add missing support of the event `$events/delivery_dropped` into the rule engine test API `rule_test`. + - Protect `mria:join/1,2` with a global lock to prevent conflicts between two nodes trying to join each other simultaneously [Mria PR](https://github.com/emqx/mria/pull/137) -- [#10747](https://github.com/emqx/emqx/pull/10747) Refactor date and time functions, `format_date` and `date_to_unix_ts`, in the rule engine to fix the implementation problem. + - Implement new function `mria:sync_transaction/4,3,2`, which blocks the caller until a transaction is imported to the local node (if the local node is a replicant, otherwise, it behaves exactly the same as `mria:transaction/3,2`) [Mria PR](https://github.com/emqx/mria/pull/136) -- [#10755](https://github.com/emqx/emqx/pull/10755) Fixed data bridge resource update race condition. + - Optimize `mria:running_nodes/0` [Mria PR](https://github.com/emqx/mria/pull/135) - In the 'delete + create' process for EMQX resource updates, - long bridge creation times could cause dashboard request timeouts. - If a bridge resource update was initiated before completion of its creation, - it led to an erroneous deletion from the runtime, despite being present in the config file. + - Optimize `mria:ro_transaction/2` when called on a replicant node [Mria PR](https://github.com/emqx/mria/pull/134). - This fix addresses the race condition in bridge resource updates, - ensuring the accurate identification and addition of new resources, - maintaining consistency between runtime and configuration file statuses. +- [#10518](https://github.com/emqx/emqx/pull/10518) Added the following fixes and features in Mria: -- [#10760](https://github.com/emqx/emqx/pull/10760) Fix Internal Error 500 that occurred sometimes when bridge statistics page was updated while a node was (re)joining the cluster. + - Call `mria_rlog:role/1` safely in mria_membership to ensure that mria_membership gen_server won't crash if RPC to another node fails [Mria PR](https://github.com/emqx/mria/pull/139) -- [#10761](https://github.com/emqx/emqx/pull/10761) Fixing the issue where the default value of SSL certificate for Dashboard Listener was not correctly interpolated, which caused HTTPS to be inaccessible when verify_peer and cacertfile were using the default configuration. + - Add an extra field to `?rlog_sync` table to facilitate extending this functionality in future [Mria PR](https://github.com/emqx/mria/pull/138). -- [#10801](https://github.com/emqx/emqx/pull/10801) Avoid duplicated percent decode the topic name in API `/topics/{topic}` and `/topics`. +- [#10556](https://github.com/emqx/emqx/pull/10556) Wrapped potentially sensitive data in `emqx_connector_http` if `Authorization` headers are being passed at initialization. -- [#10817](https://github.com/emqx/emqx/pull/10817) Fix the error of not being able to configure `auto_restart_interval` as infinity +- [#10571](https://github.com/emqx/emqx/pull/10571) Stopped emitting useless crash report when EMQX stops. -- [#10672](https://github.com/emqx/emqx/pull/10672) Fix the issue where the lack of a default value for ssl_options in listeners results in startup failure. - For example, such command(`EMQX_LISTENERS__WSS__DEFAULT__BIND='0.0.0.0:8089' ./bin/emqx console`) would have caused a crash before. +- [#10659](https://github.com/emqx/emqx/pull/10659) Fixed the issue where EMQX cannot start when `sysmon.os.mem_check_interval` is disabled. -- [#10738](https://github.com/emqx/emqx/pull/10738) Add support for the `Supertable` and `Create Tables Automatically` features of TDEngine to its data bridge. - Before this fix, an insert with a supertable in the template will fail, like this: - `insert into ${clientid} using msg TAGS (${clientid}) values (${ts},${msg})`. +- [#10717](https://github.com/emqx/emqx/pull/10717) Fixed an issue where the buffering layer processes could use a lot of CPU when inflight window is full. -- [#10741](https://github.com/emqx/emqx/pull/10741) Fix password leaking on stacktrace for Oracle Database. +- [#10724](https://github.com/emqx/emqx/pull/10724) A summary has been added for all endpoints in the HTTP API documentation (accessible at "http://:18083/api-docs"). + +- [#10726](https://github.com/emqx/emqx/pull/10726) Health Check Interval and Auto Restart Interval now support the range from 1ms to 1 hour. + +- [#10728](https://github.com/emqx/emqx/pull/10728) Fixed an issue where the rule engine was unable to access variables exported by `FOREACH` - `DO` clause. + + Given a payload: `{"date": "2023-05-06", "array": ["a"]}`, as well as the following SQL statement: + + `FOREACH payload.date as date, payload.array as elem DO date, elem FROM "t/#" -- {"date": "2023-05-06", "array": ["a"]}` + + Prior to the fix, the `date` variable exported by `FOREACH` could not be accessed in the `DO` clause of the above SQL, resulting in the following output for the SQL statement: `[{"elem": "a","date": "undefined"}]`. + +- [#10742](https://github.com/emqx/emqx/pull/10742) Correctness check of the rules is enforced before saving the authorization file source. Previously, Saving wrong rules could lead to EMQX restart failure. + +- [#10743](https://github.com/emqx/emqx/pull/10743) Fixed an issue where trying to get bridge info or metrics could result in a crash when a node is joining a cluster. + +- [#10755](https://github.com/emqx/emqx/pull/10755) Fixed data bridge resource update race condition. + + In the 'delete + create' process for EMQX resource updates, long bridge creation times could cause dashboard request timeouts. If a bridge resource update was initiated before completion of its creation, it led to an erroneous deletion from the runtime, despite being present in the config file. + + This fix addresses the race condition in bridge resource updates, ensuring the accurate identification and addition of new resources, and maintaining consistency between runtime and configuration file statuses. + +- [#10761](https://github.com/emqx/emqx/pull/10761) Fixed the issue where the default value of SSL certificate for Dashboard Listener was not correctly interpolated, which caused HTTPS to be inaccessible when `verify_peer` and `cacertfile` were using the default configuration. + +- [#10672](https://github.com/emqx/emqx/pull/10672) Fixed the issue where the lack of a default value for `ssl_options` in listeners results in startup failure. For example, such command(`EMQX_LISTENERS__WSS__DEFAULT__BIND='0.0.0.0:8089' ./bin/emqx console`) would have caused a crash before. + +- [#10738](https://github.com/emqx/emqx/pull/10738) TDEngine data bridge now supports "Supertable" and "Create Tables Automatically". Before this fix, an insert with a supertable in the template will fail, like this: + + - `insert into ${clientid} using msg TAGS (${clientid}) values (${ts},${msg})`. + +- [#10746](https://github.com/emqx/emqx/pull/10746) Add missing support of the event `$events/delivery_dropped` into the rule engine test API `rule_test`. + +- [#10747](https://github.com/emqx/emqx/pull/10747) Ported some time formating fixes in Rule-Engine functions from version 4.4. + +- [#10760](https://github.com/emqx/emqx/pull/10760) Fix "internal error 500" when getting bridge statistics page while a node is joining the cluster. + +- [#10801](https://github.com/emqx/emqx/pull/10801) Avoid double percent-decode for topic name in API `/topics/{topic}` and `/topics`. + +- [#10817](https://github.com/emqx/emqx/pull/10817) Fix a config value handling for bridge resource option `auto_restart_interval`, now it can be set to `infinity`. From 687664efdbe1f9fae240fe11bbf41940aa88ebb5 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 26 May 2023 15:49:25 +0200 Subject: [PATCH 19/20] ci: delete otp 24 from emqx app standalone job matrix --- .github/workflows/run_emqx_app_tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 20429392d..ffed8d0c8 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -14,7 +14,6 @@ jobs: builder: - 5.0-35 otp: - - 24.3.4.2-3 - 25.1.2-3 # no need to use more than 1 version of Elixir, since tests # run using only Erlang code. This is needed just to specify From 36e268c93317efc3358504396c373e22e1a265bf Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 26 May 2023 16:05:37 +0200 Subject: [PATCH 20/20] chore: bump app versions --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_authz/src/emqx_authz.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src | 2 +- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_connector/src/emqx_connector.app.src | 2 +- apps/emqx_ft/src/emqx_ft.app.src | 2 +- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_management/src/emqx_management.app.src | 2 +- apps/emqx_resource/src/emqx_resource.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- apps/emqx_s3/src/emqx_s3.app.src | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index be68b438f..2a2b3c323 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.26"}, + {vsn, "5.0.27"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index 4856008e6..7620f5548 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.20"}, + {vsn, "0.1.21"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index d2bf0f0c2..125f53c81 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.19"}, + {vsn, "0.1.20"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src index 1665548ae..f50c34391 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_pulsar, [ {description, "EMQX Pulsar Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index c31a16b9b..3a5ebc77a 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.20"}, + {vsn, "0.1.21"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 76c3e8bb9..e9f79723a 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.23"}, + {vsn, "0.1.24"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src index 058fe984a..c503ab8e0 100644 --- a/apps/emqx_ft/src/emqx_ft.app.src +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -1,6 +1,6 @@ {application, emqx_ft, [ {description, "EMQX file transfer over MQTT"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_ft_app, []}}, {applications, [ diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 26dab4c9c..487e71a06 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.17"}, + {vsn, "0.1.18"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, emqx, emqx_authn, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index f51a83923..31c719a33 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.22"}, + {vsn, "5.0.23"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 7c1b5ad2a..25ce1b79e 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.16"}, + {vsn, "0.1.17"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 8dc78958a..c6f94f5ea 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.17"}, + {vsn, "5.0.18"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]}, diff --git a/apps/emqx_s3/src/emqx_s3.app.src b/apps/emqx_s3/src/emqx_s3.app.src index 7864ffb29..6d0518769 100644 --- a/apps/emqx_s3/src/emqx_s3.app.src +++ b/apps/emqx_s3/src/emqx_s3.app.src @@ -1,6 +1,6 @@ {application, emqx_s3, [ {description, "EMQX S3"}, - {vsn, "5.0.6"}, + {vsn, "5.0.7"}, {modules, []}, {registered, [emqx_s3_sup]}, {applications, [ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index ae922ecee..4a52b2c35 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.13"}, + {vsn, "0.1.14"}, {registered, []}, {applications, [ kernel,