From 25cdc2a04bbd336df330cd7a632c1181940f0196 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 21 Feb 2024 09:57:05 -0300 Subject: [PATCH 01/25] chore: bump version to 5.5.1-rc.1 --- apps/emqx/include/emqx_release.hrl | 4 ++-- deploy/charts/emqx-enterprise/Chart.yaml | 4 ++-- deploy/charts/emqx/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 7886ec786..7cf538edc 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,10 +32,10 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.5.0"). +-define(EMQX_RELEASE_CE, "5.5.1-rc.1"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.5.0"). +-define(EMQX_RELEASE_EE, "5.5.1-rc.1"). %% The HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index b870d1123..c0b81df81 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.5.0 +version: 5.5.1-rc.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.5.0 +appVersion: 5.5.1-rc.1 diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 41f0b37b6..a1316f9cd 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.5.0 +version: 5.5.1-rc.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.5.0 +appVersion: 5.5.1-rc.1 From b60b19a29a1c8f630ed8d64454db952e8f59acb5 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 23 Feb 2024 10:17:24 -0300 Subject: [PATCH 02/25] fix(connectors): redact authorization headers when creating/updating connectors Fixes https://emqx.atlassian.net/browse/EMQX-11895 --- .../src/emqx_bridge_http_connector.erl | 2 +- apps/emqx_connector/src/emqx_connector.app.src | 2 +- .../src/emqx_connector_resource.erl | 16 +++++++++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index ac9b18ace..f62fc9d3f 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -48,7 +48,7 @@ ]). %% for other http-like connectors. --export([redact_request/1]). +-export([redact_request/1, is_sensitive_key/1]). -export([validate_method/1, join_paths/2]). diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 09adf9977..4622e41bb 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.38"}, + {vsn, "0.1.39"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index 74c167b32..4d753d477 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -137,7 +137,7 @@ create(Type, Name, Conf0, Opts) -> msg => "create connector", type => Type, name => Name, - config => emqx_utils:redact(Conf0) + config => redact(Conf0, Type) }), TypeBin = bin(Type), ResourceId = resource_id(Type, Name), @@ -174,7 +174,7 @@ update(Type, Name, {OldConf, Conf}, Opts) -> msg => "update connector", type => Type, name => Name, - config => emqx_utils:redact(Conf) + config => redact(Conf, Type) }), case recreate(Type, Name, Conf, Opts) of {ok, _} -> @@ -184,7 +184,7 @@ update(Type, Name, {OldConf, Conf}, Opts) -> msg => "updating_a_non_existing_connector", type => Type, name => Name, - config => emqx_utils:redact(Conf) + config => redact(Conf, Type) }), create(Type, Name, Conf, Opts); {error, Reason} -> @@ -378,3 +378,13 @@ override_start_after_created(Config, Opts) -> set_no_buffer_workers(Opts) -> Opts#{spawn_buffer_workers => false}. + +%% TODO: introduce a formal callback? +redact(Conf, Type) when + Type =:= http; + Type =:= <<"http">> +-> + %% CE bridge + emqx_utils:redact(Conf, fun emqx_bridge_http_connector:is_sensitive_key/1); +redact(Conf, _Type) -> + emqx_utils:redact(Conf). From a3e81c50392d9f623d5054517c4a8aa6194f8e1b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 24 Feb 2024 08:59:51 +0100 Subject: [PATCH 03/25] chore: prepare for release 5.5.1-rc.2 --- apps/emqx/include/emqx_release.hrl | 4 ++-- deploy/charts/emqx-enterprise/Chart.yaml | 4 ++-- deploy/charts/emqx/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 7cf538edc..2c687cd48 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,10 +32,10 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.5.1-rc.1"). +-define(EMQX_RELEASE_CE, "5.5.1-rc.2"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.5.1-rc.1"). +-define(EMQX_RELEASE_EE, "5.5.1-rc.2"). %% The HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index c0b81df81..c1fbef623 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.5.1-rc.1 +version: 5.5.1-rc.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.5.1-rc.1 +appVersion: 5.5.1-rc.2 diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index a1316f9cd..3b0a12d3b 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.5.1-rc.1 +version: 5.5.1-rc.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.5.1-rc.1 +appVersion: 5.5.1-rc.2 From d8032f47ca1c184372a6f7368ab4187e4ebff939 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 26 Feb 2024 17:21:03 -0300 Subject: [PATCH 04/25] fix: redact all headers from logs Fixes https://emqx.atlassian.net/browse/EMQX-11904 Since headers are usually used for authentication and the headers used for that are very flexible, we redact all headers from logs to avoid leaking anything. --- apps/emqx_audit/test/emqx_audit_api_SUITE.erl | 2 +- apps/emqx_utils/src/emqx_utils.app.src | 2 +- apps/emqx_utils/src/emqx_utils.erl | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl index 2f401e7a8..f1f4f2628 100644 --- a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl +++ b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl @@ -88,7 +88,7 @@ t_http_api(_) -> <<"method">> := <<"put">>, <<"body">> := #{<<"mqtt">> := #{<<"max_qos_allowed">> := 1}}, <<"bindings">> := _, - <<"headers">> := #{<<"authorization">> := <<"******">>} + <<"headers">> := "******" }, <<"http_status_code">> := 200, <<"operation_result">> := <<"success">>, diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index 766b25da6..8fdade473 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -2,7 +2,7 @@ {application, emqx_utils, [ {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! - {vsn, "5.0.15"}, + {vsn, "5.0.16"}, {modules, [ emqx_utils, emqx_utils_api, diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl index 0eeef2e5e..be9f99923 100644 --- a/apps/emqx_utils/src/emqx_utils.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -717,6 +717,9 @@ is_sensitive_key(<<"jwt">>) -> true; is_sensitive_key(authorization) -> true; is_sensitive_key("authorization") -> true; is_sensitive_key(<<"authorization">>) -> true; +is_sensitive_key(headers) -> true; +is_sensitive_key("headers") -> true; +is_sensitive_key(<<"headers">>) -> true; is_sensitive_key(bind_password) -> true; is_sensitive_key("bind_password") -> true; is_sensitive_key(<<"bind_password">>) -> true; @@ -879,6 +882,7 @@ redact_test_() -> secret_key, secret_access_key, security_token, + headers, token, bind_password ], From a76415c4f6d304a24b5130f2062fb09db59da006 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 27 Feb 2024 14:15:27 +0800 Subject: [PATCH 05/25] fix(mgmt): sub/unsub a share subscription to the client via http api - `/clients/:clientid/subscribe` - `/clients/:clientid/subscribe/bulk` - `/clients/:clientid/unsubscribe` - `/clients/:clientid/unsubscribe/bulk` --- apps/emqx/src/emqx_topic.erl | 7 ++++--- apps/emqx_management/src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_mgmt_api_clients.erl | 7 +++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/emqx/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl index bc6946a43..30744b300 100644 --- a/apps/emqx/src/emqx_topic.erl +++ b/apps/emqx/src/emqx_topic.erl @@ -269,10 +269,11 @@ do_join(_TopicAcc, [C | Words]) when ?MULTI_LEVEL_WILDCARD_NOT_LAST(C, Words) -> do_join(TopicAcc, [Word | Words]) -> do_join(<>, Words). --spec parse(topic() | {topic(), map()}) -> {topic() | share(), map()}. -parse(TopicFilter) when is_binary(TopicFilter) -> +-spec parse(TF | {TF, map()}) -> {TF, map()} when + TF :: topic() | share(). +parse(TopicFilter) when ?IS_TOPIC(TopicFilter) -> parse(TopicFilter, #{}); -parse({TopicFilter, Options}) when is_binary(TopicFilter) -> +parse({TopicFilter, Options}) when ?IS_TOPIC(TopicFilter) -> parse(TopicFilter, Options). -spec parse(topic() | share(), map()) -> {topic() | share(), map()}. diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index a44983596..ad9e12b90 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.37"}, + {vsn, "5.0.38"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [ diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index f394ffefa..ef4fd2e63 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -781,10 +781,13 @@ subscribe_batch(#{clientid := ClientID, topics := Topics}) -> end. unsubscribe(#{clientid := ClientID, topic := Topic}) -> + {NTopic, _} = emqx_topic:parse(Topic), case do_unsubscribe(ClientID, Topic) of {error, channel_not_found} -> {404, ?CLIENTID_NOT_FOUND}; - {unsubscribe, [{Topic, #{}}]} -> + {unsubscribe, [{UnSubedT, #{}}]} when + (UnSubedT =:= NTopic) orelse (UnSubedT =:= Topic) + -> {204} end. @@ -809,7 +812,7 @@ do_subscribe(ClientID, Topic0, Options) -> {subscribe, Subscriptions, Node} -> case proplists:is_defined(Topic, Subscriptions) of true -> - {ok, Options#{node => Node, clientid => ClientID, topic => Topic}}; + {ok, Options#{node => Node, clientid => ClientID, topic => Topic0}}; false -> {error, unknow_error} end From c1da7449feb811cf2dfff3cc224995a4e6268536 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 27 Feb 2024 15:54:59 +0800 Subject: [PATCH 06/25] test(mgmt_api): bulk/unbulk sub/unsub shared topic filter --- .../test/emqx_mgmt_api_clients_SUITE.erl | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 6daf918f1..0d9e8eef6 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -430,6 +430,62 @@ t_client_id_not_found(_Config) -> {error, {Http, _, Body}}, PostFun(post, PathFun(["unsubscribe", "bulk"]), [UnsubBody]) ). +t_subscribe_shared_topic(_Config) -> + ClientId = <<"client_subscribe_shared">>, + + {ok, C} = emqtt:start_link(#{clientid => ClientId}), + {ok, _} = emqtt:connect(C), + {ok, _, _} = emqtt:subscribe(C, <<"topic/1">>, 1), + + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + + Http200 = {"HTTP/1.1", 200, "OK"}, + Http204 = {"HTTP/1.1", 204, "No Content"}, + + PathFun = fun(Suffix) -> + emqx_mgmt_api_test_util:api_path(["clients", "client_subscribe_shared"] ++ Suffix) + end, + + PostFun = fun(Method, Path, Data) -> + emqx_mgmt_api_test_util:request_api( + Method, Path, "", AuthHeader, Data, #{return_all => true} + ) + end, + + SharedT = <<"$share/group/testtopic">>, + NonSharedT = <<"t/1">>, + + SubBodyFun = fun(T) -> #{topic => T, qos => 1, nl => 1, rh => 1} end, + UnSubBodyFun = fun(T) -> #{topic => T} end, + + %% Client Subscribe + ?assertMatch( + {ok, {Http200, _, _}}, + PostFun(post, PathFun(["subscribe"]), SubBodyFun(SharedT)) + ), + ?assertMatch( + {ok, {Http200, _, _}}, + PostFun( + post, + PathFun(["subscribe", "bulk"]), + [SubBodyFun(T) || T <- [SharedT, NonSharedT]] + ) + ), + + %% Client Unsubscribe + ?assertMatch( + {ok, {Http204, _, _}}, + PostFun(post, PathFun(["unsubscribe"]), UnSubBodyFun(SharedT)) + ), + ?assertMatch( + {ok, {Http204, _, _}}, + PostFun( + post, + PathFun(["unsubscribe", "bulk"]), + [UnSubBodyFun(T) || T <- [SharedT, NonSharedT]] + ) + ). + time_string_to_epoch_millisecond(DateTime) -> time_string_to_epoch(DateTime, millisecond). From c1928f874e9e5a1196257e7b6d0faf475e7a7df5 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 27 Feb 2024 15:59:56 +0800 Subject: [PATCH 07/25] chore: add change log --- changes/ce/fix-12598.en.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 changes/ce/fix-12598.en.md diff --git a/changes/ce/fix-12598.en.md b/changes/ce/fix-12598.en.md new file mode 100644 index 000000000..fd077d492 --- /dev/null +++ b/changes/ce/fix-12598.en.md @@ -0,0 +1,9 @@ +Fixed an issue that unable to subscribe or unsubscribe a shared topic filter via HTTP API. + +Releated APIs: + +- `/clients/:clientid/subscribe` +- `/clients/:clientid/subscribe/bulk` + +- `/clients/:clientid/unsubscribe` +- `/clients/:clientid/unsubscribe/bulk` From 5c69500c52602574fbe925cec16c1ebb4c421695 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 27 Feb 2024 15:47:46 +0800 Subject: [PATCH 08/25] fix(ldap): fix that logs of eldap will never be logged --- apps/emqx_ldap/src/emqx_ldap.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index 04b61918a..0758d0477 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -312,11 +312,12 @@ do_ldap_query( {error, {unrecoverable_error, Reason}} end. -log(Level, Format, Args) -> +%% Note: the value of the `_Level` here always is 2 +log(_Level, Format, Args) -> ?SLOG( - Level, + info, #{ - msg => "ldap_log", + msg => "eldap_info", log => io_lib:format(Format, Args) } ). From f3237a1bf78a938bc0c8278b90271154969e3749 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 27 Feb 2024 16:08:47 +0800 Subject: [PATCH 09/25] chore: bump emqx_ldap version --- apps/emqx_ldap/src/emqx_ldap.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_ldap/src/emqx_ldap.app.src b/apps/emqx_ldap/src/emqx_ldap.app.src index 546c9975c..3598d317c 100644 --- a/apps/emqx_ldap/src/emqx_ldap.app.src +++ b/apps/emqx_ldap/src/emqx_ldap.app.src @@ -1,6 +1,6 @@ {application, emqx_ldap, [ {description, "EMQX LDAP Connector"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {applications, [ kernel, From c6b1102b2b5aab3ee3b2ead099eae2d897e21c34 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 27 Feb 2024 17:51:39 +0800 Subject: [PATCH 10/25] fix(mgmt_client): `nl` not allowed for shared-sub --- apps/emqx/src/emqx_topic.erl | 4 +++ .../src/emqx_mgmt_api_clients.erl | 35 +++++++++++++------ 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/apps/emqx/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl index 30744b300..b48c05af5 100644 --- a/apps/emqx/src/emqx_topic.erl +++ b/apps/emqx/src/emqx_topic.erl @@ -283,6 +283,10 @@ parse(#share{topic = Topic = <>}, _Options) -> error({invalid_topic_filter, Topic}); parse(#share{topic = Topic = <>}, _Options) -> error({invalid_topic_filter, Topic}); +parse(#share{} = T, #{nl := 1} = _Options) -> + %% Protocol Error and Should Disconnect + %% MQTT-5.0 [MQTT-3.8.3-4] and [MQTT-4.13.1-1] + error({invalid_subopts_nl, maybe_format_share(T)}); parse(<>, Options) -> parse(#share{group = <>, topic = Topic}, Options); parse(TopicFilter = <>, Options) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index ef4fd2e63..e6a9b9555 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -758,6 +758,12 @@ subscribe(#{clientid := ClientID, topic := Topic} = Sub) -> case do_subscribe(ClientID, Topic, Opts) of {error, channel_not_found} -> {404, ?CLIENTID_NOT_FOUND}; + {error, invalid_subopts_nl} -> + {400, #{ + code => <<"INVALID_PARAMETER">>, + message => + <<"Invalid Subscribe options: `no_local` not allowed for shared-sub. See [MQTT-3.8.3-4]">> + }}; {error, Reason} -> Message = list_to_binary(io_lib:format("~p", [Reason])), {500, #{code => <<"UNKNOWN_ERROR">>, message => Message}}; @@ -804,18 +810,25 @@ unsubscribe_batch(#{clientid := ClientID, topics := Topics}) -> %% internal function do_subscribe(ClientID, Topic0, Options) -> - {Topic, Opts} = emqx_topic:parse(Topic0, Options), - TopicTable = [{Topic, Opts}], - case emqx_mgmt:subscribe(ClientID, TopicTable) of - {error, Reason} -> - {error, Reason}; - {subscribe, Subscriptions, Node} -> - case proplists:is_defined(Topic, Subscriptions) of - true -> - {ok, Options#{node => Node, clientid => ClientID, topic => Topic0}}; - false -> - {error, unknow_error} + try emqx_topic:parse(Topic0, Options) of + {Topic, Opts} -> + TopicTable = [{Topic, Opts}], + case emqx_mgmt:subscribe(ClientID, TopicTable) of + {error, Reason} -> + {error, Reason}; + {subscribe, Subscriptions, Node} -> + case proplists:is_defined(Topic, Subscriptions) of + true -> + {ok, Options#{node => Node, clientid => ClientID, topic => Topic0}}; + false -> + {error, unknow_error} + end end + catch + error:{invalid_subopts_nl, _} -> + {error, invalid_subopts_nl}; + _:Reason -> + {error, Reason} end. -spec do_unsubscribe(emqx_types:clientid(), emqx_types:topic()) -> From 2ded75123708fb82dc4cdb53e48ebc464b938b18 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 27 Feb 2024 18:06:10 +0800 Subject: [PATCH 11/25] test: `nl` not allowed for shared-sub --- .../test/emqx_mgmt_api_clients_SUITE.erl | 79 ++++++++++++++++++- 1 file changed, 76 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 0d9e8eef6..6d919a38a 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -17,8 +17,11 @@ -compile(export_all). -compile(nowarn_export_all). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_router.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("emqx/include/asserts.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). @@ -435,7 +438,10 @@ t_subscribe_shared_topic(_Config) -> {ok, C} = emqtt:start_link(#{clientid => ClientId}), {ok, _} = emqtt:connect(C), - {ok, _, _} = emqtt:subscribe(C, <<"topic/1">>, 1), + + ClientPuber = <<"publish_client">>, + {ok, PC} = emqtt:start_link(#{clientid => ClientPuber}), + {ok, _} = emqtt:connect(PC), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), @@ -453,11 +459,12 @@ t_subscribe_shared_topic(_Config) -> end, SharedT = <<"$share/group/testtopic">>, - NonSharedT = <<"t/1">>, + NonSharedT = <<"t/#">>, - SubBodyFun = fun(T) -> #{topic => T, qos => 1, nl => 1, rh => 1} end, + SubBodyFun = fun(T) -> #{topic => T, qos => 1, nl => 0, rh => 1} end, UnSubBodyFun = fun(T) -> #{topic => T} end, + %% ==================== %% Client Subscribe ?assertMatch( {ok, {Http200, _, _}}, @@ -472,6 +479,36 @@ t_subscribe_shared_topic(_Config) -> ) ), + %% assert subscription + ?assertMatch( + [ + {_, #share{group = <<"group">>, topic = <<"testtopic">>}}, + {_, <<"t/#">>} + ], + ets:tab2list(?SUBSCRIPTION) + ), + + ?assertMatch( + [ + {{#share{group = <<"group">>, topic = <<"testtopic">>}, _}, #{ + nl := 0, qos := 1, rh := 1, rap := 0 + }}, + {{<<"t/#">>, _}, #{nl := 0, qos := 1, rh := 1, rap := 0}} + ], + ets:tab2list(?SUBOPTION) + ), + ?assertMatch( + [{emqx_shared_subscription, <<"group">>, <<"testtopic">>, _}], + ets:tab2list(emqx_shared_subscription) + ), + + %% assert subscription virtual + _ = emqtt:publish(PC, <<"testtopic">>, <<"msg1">>, [{qos, 0}]), + ?assertReceive({publish, #{topic := <<"testtopic">>, payload := <<"msg1">>}}), + _ = emqtt:publish(PC, <<"t/1">>, <<"msg2">>, [{qos, 0}]), + ?assertReceive({publish, #{topic := <<"t/1">>, payload := <<"msg2">>}}), + + %% ==================== %% Client Unsubscribe ?assertMatch( {ok, {Http204, _, _}}, @@ -484,6 +521,42 @@ t_subscribe_shared_topic(_Config) -> PathFun(["unsubscribe", "bulk"]), [UnSubBodyFun(T) || T <- [SharedT, NonSharedT]] ) + ), + + %% assert subscription + ?assertEqual([], ets:tab2list(?SUBSCRIPTION)), + ?assertEqual([], ets:tab2list(?SUBOPTION)), + ?assertEqual([], ets:tab2list(emqx_shared_subscription)), + + %% assert subscription virtual + _ = emqtt:publish(PC, <<"testtopic">>, <<"msg3">>, [{qos, 0}]), + _ = emqtt:publish(PC, <<"t/1">>, <<"msg4">>, [{qos, 0}]), + ?assertNotReceive({publish, #{topic := <<"testtopic">>, payload := <<"msg3">>}}), + ?assertNotReceive({publish, #{topic := <<"t/1">>, payload := <<"msg4">>}}). + +t_subscribe_shared_topic_nl(_Config) -> + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Http400 = {"HTTP/1.1", 400, "Bad Request"}, + Body = + "{\"code\":\"INVALID_PARAMETER\"," + "\"message\":\"Invalid Subscribe options: `no_local` not allowed for shared-sub. See [MQTT-3.8.3-4]\"}", + ClientId = <<"client_subscribe_shared">>, + + {ok, C} = emqtt:start_link(#{clientid => ClientId}), + {ok, _} = emqtt:connect(C), + + PathFun = fun(Suffix) -> + emqx_mgmt_api_test_util:api_path(["clients", "client_subscribe_shared"] ++ Suffix) + end, + PostFun = fun(Method, Path, Data) -> + emqx_mgmt_api_test_util:request_api( + Method, Path, "", AuthHeader, Data, #{return_all => true} + ) + end, + T = <<"$share/group/testtopic">>, + ?assertMatch( + {error, {Http400, _, Body}}, + PostFun(post, PathFun(["subscribe"]), #{topic => T, qos => 1, nl => 1, rh => 1}) ). time_string_to_epoch_millisecond(DateTime) -> From 37026fa94f095864880d34eada0caef99e679a79 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 27 Feb 2024 21:03:53 +0800 Subject: [PATCH 12/25] chore: update change --- changes/ce/fix-12601.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-12601.en.md diff --git a/changes/ce/fix-12601.en.md b/changes/ce/fix-12601.en.md new file mode 100644 index 000000000..076946f37 --- /dev/null +++ b/changes/ce/fix-12601.en.md @@ -0,0 +1 @@ +Fixed that the logs of LDAP driver would never be logged, now all of them are logged with `info` level. From 042a84c30f9c6f968b9cd40dcd3115aa91636da3 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 28 Feb 2024 11:33:27 +0800 Subject: [PATCH 13/25] fix(prom): api crash when tls cert file non existed --- .../src/emqx_prometheus.app.src | 2 +- apps/emqx_prometheus/src/emqx_prometheus.erl | 20 +++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index 9e9952d6c..b8c3d8790 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.19"}, + {vsn, "5.0.20"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_auth, emqx_resource, emqx_management]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 59241bd02..309a794fb 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -883,13 +883,17 @@ gen_point(Type, Name, Path) -> %% TODO: cert manager for more generic utils functions cert_expiry_at_from_path(Path0) -> Path = emqx_schema:naive_env_interpolation(Path0), - {ok, PemBin} = file:read_file(Path), - [CertEntry | _] = public_key:pem_decode(PemBin), - Cert = public_key:pem_entry_decode(CertEntry), - %% TODO: Not fully tested for all certs type - {'utcTime', NotAfterUtc} = - Cert#'Certificate'.'tbsCertificate'#'TBSCertificate'.validity#'Validity'.'notAfter', - utc_time_to_epoch(NotAfterUtc). + case file:read_file(Path) of + {ok, PemBin} -> + [CertEntry | _] = public_key:pem_decode(PemBin), + Cert = public_key:pem_entry_decode(CertEntry), + %% TODO: Not fully tested for all certs type + {'utcTime', NotAfterUtc} = + Cert#'Certificate'.'tbsCertificate'#'TBSCertificate'.validity#'Validity'.'notAfter', + utc_time_to_epoch(NotAfterUtc); + {error, _} -> + 0 + end. utc_time_to_epoch(UtcTime) -> date_to_expiry_epoch(utc_time_to_datetime(UtcTime)). @@ -898,7 +902,7 @@ utc_time_to_datetime(Str) -> {ok, [Year, Month, Day, Hour, Minute, Second], _} = io_lib:fread( "~2d~2d~2d~2d~2d~2dZ", Str ), - %% Alwoys Assuming YY is in 2000 + %% Always Assuming YY is in 2000 {{2000 + Year, Month, Day}, {Hour, Minute, Second}}. %% 62167219200 =:= calendar:datetime_to_gregorian_seconds({{1970, 1, 1}, {0, 0, 0}}). From 812e8ef314b3c3e78691ba9ea8d56bae4459676d Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 28 Feb 2024 14:29:46 +0800 Subject: [PATCH 14/25] fix: try-catch for unknow reason to inhibit api crash --- apps/emqx_prometheus/src/emqx_prometheus.erl | 36 +++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 309a794fb..8b34cc9eb 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -883,15 +883,33 @@ gen_point(Type, Name, Path) -> %% TODO: cert manager for more generic utils functions cert_expiry_at_from_path(Path0) -> Path = emqx_schema:naive_env_interpolation(Path0), - case file:read_file(Path) of - {ok, PemBin} -> - [CertEntry | _] = public_key:pem_decode(PemBin), - Cert = public_key:pem_entry_decode(CertEntry), - %% TODO: Not fully tested for all certs type - {'utcTime', NotAfterUtc} = - Cert#'Certificate'.'tbsCertificate'#'TBSCertificate'.validity#'Validity'.'notAfter', - utc_time_to_epoch(NotAfterUtc); - {error, _} -> + try + case file:read_file(Path) of + {ok, PemBin} -> + [CertEntry | _] = public_key:pem_decode(PemBin), + Cert = public_key:pem_entry_decode(CertEntry), + %% TODO: Not fully tested for all certs type + {'utcTime', NotAfterUtc} = + Cert#'Certificate'.'tbsCertificate'#'TBSCertificate'.validity#'Validity'.'notAfter', + utc_time_to_epoch(NotAfterUtc); + {error, Reason} -> + ?SLOG(error, #{ + msg => "read_cert_file_failed", + path => Path0, + resolved_path => Path, + reason => Reason + }), + 0 + end + catch + E:R -> + ?SLOG(error, #{ + msg => "obtain_cert_expiry_time_failed", + error => E, + reason => R, + path => Path0, + resolved_path => Path + }), 0 end. From 5f125047f583c9a9db69328677f840de385406b1 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 28 Feb 2024 14:50:09 +0800 Subject: [PATCH 15/25] refactor: remove unnecessary lists fold for `cerfile` --- apps/emqx_prometheus/src/emqx_prometheus.erl | 38 ++++++++------------ 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 8b34cc9eb..399232c40 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -830,8 +830,7 @@ cert_data(undefined) -> cert_data(AllListeners) -> Points = lists:foldl( fun(ListenerType, PointsAcc) -> - PointsAcc ++ - points_of_listeners(ListenerType, AllListeners) + lists:append(PointsAcc, points_of_listeners(ListenerType, AllListeners)) end, _PointsInitAcc = [], ?LISTENER_TYPES @@ -843,41 +842,34 @@ cert_data(AllListeners) -> points_of_listeners(Type, AllListeners) -> do_points_of_listeners(Type, maps:get(Type, AllListeners, undefined)). --define(CERT_TYPES, [certfile]). - --spec do_points_of_listeners(Type, TypeOfListeners) -> +-spec do_points_of_listeners(Type, Listeners) -> [_Point :: {[{LabelKey, LabelValue}], Epoch}] when Type :: ssl | wss | quic, - TypeOfListeners :: #{ListenerName :: atom() => ListenerConf :: map()} | undefined, + Listeners :: #{ListenerName :: atom() => ListenerConf :: map()} | undefined, LabelKey :: atom(), LabelValue :: atom(), Epoch :: non_neg_integer(). do_points_of_listeners(_, undefined) -> []; -do_points_of_listeners(ListenerType, TypeOfListeners) -> +do_points_of_listeners(Type, Listeners) -> lists:foldl( fun(Name, PointsAcc) -> - lists:foldl( - fun(CertType, AccIn) -> - case - emqx_utils_maps:deep_get( - [Name, ssl_options, CertType], TypeOfListeners, undefined - ) - of - undefined -> AccIn; - Path -> [gen_point(ListenerType, Name, Path) | AccIn] - end - end, - [], - ?CERT_TYPES - ) ++ PointsAcc + case + emqx_utils_maps:deep_get( + [Name, ssl_options, certfile], Listeners, undefined + ) + of + undefined -> PointsAcc; + Path -> [gen_point_cert_expiry_at(Type, Name, Path) | PointsAcc] + end end, [], - maps:keys(TypeOfListeners) + %% listener names + maps:keys(Listeners) ). -gen_point(Type, Name, Path) -> +gen_point_cert_expiry_at(Type, Name, Path) -> {[{listener_type, Type}, {listener_name, Name}], cert_expiry_at_from_path(Path)}. %% TODO: cert manager for more generic utils functions From 0e12503a8d69cc1c2a58ab4d182c82bd3cfdc44e Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 28 Feb 2024 15:25:54 +0800 Subject: [PATCH 16/25] chore: add fix change log --- changes/ce/fix-12606.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-12606.en.md diff --git a/changes/ce/fix-12606.en.md b/changes/ce/fix-12606.en.md new file mode 100644 index 000000000..ab928848d --- /dev/null +++ b/changes/ce/fix-12606.en.md @@ -0,0 +1 @@ +Fix the issue of the endpoint `/prometheus/stats` crashing when the listener's cert file is unreadable. From 6fd04e33f556b94aa10e104d46869bf02f65d2a0 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 28 Feb 2024 15:37:48 +0800 Subject: [PATCH 17/25] fix(prom): skip cert info for disabled ssl/wss/quic listeners --- apps/emqx_prometheus/src/emqx_prometheus.erl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 399232c40..3297d12e3 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -856,10 +856,12 @@ do_points_of_listeners(Type, Listeners) -> lists:foldl( fun(Name, PointsAcc) -> case - emqx_utils_maps:deep_get( - [Name, ssl_options, certfile], Listeners, undefined - ) + emqx_utils_maps:deep_get([Name, enable], Listeners, false) andalso + emqx_utils_maps:deep_get( + [Name, ssl_options, certfile], Listeners, undefined + ) of + false -> PointsAcc; undefined -> PointsAcc; Path -> [gen_point_cert_expiry_at(Type, Name, Path) | PointsAcc] end @@ -894,11 +896,12 @@ cert_expiry_at_from_path(Path0) -> 0 end catch - E:R -> + E:R:S -> ?SLOG(error, #{ msg => "obtain_cert_expiry_time_failed", error => E, reason => R, + stacktrace => S, path => Path0, resolved_path => Path }), From 01207ef97bce6f697f9595cd7b8028230801e2be Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 28 Feb 2024 16:54:01 +0800 Subject: [PATCH 18/25] fix(iotdb): fix function clause error when there is no `payload` field --- apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl index ccf97f143..2fc8f7dd0 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl @@ -367,6 +367,8 @@ on_get_channel_status(_InstanceId, ChannelId, #{channels := Channels}) -> get_payload(#{payload := Payload}) -> Payload; get_payload(#{<<"payload">> := Payload}) -> + Payload; +get_payload(Payload) -> Payload. parse_payload(ParsedPayload) when is_map(ParsedPayload) -> @@ -694,7 +696,7 @@ render_channel_message(#{is_aligned := IsAligned} = Channel, IoTDBVsn, Message) DeviceId -> case get_data_template(Channel, Payloads) of [] -> - {error, invalid_data}; + {error, invalid_template}; DataTemplate -> case proc_data(DataTemplate, Message) of {ok, DataList} -> From ce2552712fdf101626b74f0418c7f6536fc6769d Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 28 Feb 2024 17:33:12 +0800 Subject: [PATCH 19/25] chore: bump version && update change --- apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src | 2 +- .../test/emqx_bridge_iotdb_impl_SUITE.erl | 8 ++++---- changes/ee/fix-12608.en.md | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 changes/ee/fix-12608.en.md diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src index 4fd96d5e7..86d2a93b3 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_iotdb, [ {description, "EMQX Enterprise Apache IoTDB Bridge"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {modules, [ emqx_bridge_iotdb, emqx_bridge_iotdb_connector diff --git a/apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl b/apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl index 1093993b2..edf9334b9 100644 --- a/apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl +++ b/apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl @@ -502,11 +502,11 @@ t_extract_device_id_from_rule_engine_message(Config) -> ), ok. -t_sync_invalid_data(Config) -> +t_sync_invalid_template(Config) -> emqx_bridge_v2_testlib:t_sync_query( Config, make_message_fun(iotdb_topic(Config), #{foo => bar, device_id => <<"root.sg27">>}), - is_error_check(invalid_data), + is_error_check(invalid_template), iotdb_bridge_on_query ). @@ -518,11 +518,11 @@ t_async_device_id_missing(Config) -> iotdb_bridge_on_query_async ). -t_async_invalid_data(Config) -> +t_async_invalid_template(Config) -> emqx_bridge_v2_testlib:t_async_query( Config, make_message_fun(iotdb_topic(Config), #{foo => bar, device_id => <<"root.sg27">>}), - is_error_check(invalid_data), + is_error_check(invalid_template), iotdb_bridge_on_query_async ). diff --git a/changes/ee/fix-12608.en.md b/changes/ee/fix-12608.en.md new file mode 100644 index 000000000..d4dc92b5a --- /dev/null +++ b/changes/ee/fix-12608.en.md @@ -0,0 +1 @@ +Fixed a `function_clause` error for IoTDB action when there is no `payload` field in the query data. From b9f644c3558c30f74b9a9fd6bebf3f48a5eedcbe Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 28 Feb 2024 21:46:03 +0800 Subject: [PATCH 20/25] fix(ldap): fixed that the connection to the LDAP connector could be disconnected after a period of time --- apps/emqx_ldap/src/emqx_ldap.erl | 46 ++++++++++++++------ apps/emqx_ldap/src/emqx_ldap_bind_worker.erl | 8 +++- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index 0758d0477..b40375eed 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -39,11 +39,12 @@ -export([namespace/0, roots/0, fields/1, desc/1]). --export([do_get_status/1]). +-export([do_get_status/1, get_status_with_poolname/1]). -define(LDAP_HOST_OPTIONS, #{ default_port => 389 }). +-define(REDACT_VAL, "******"). -type params_tokens() :: #{atom() => list()}. -type state() :: @@ -154,18 +155,19 @@ on_start( false -> Config2 end, + Options = [ {pool_size, PoolSize}, {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}, {options, Config3} ], - case emqx_resource_pool:start(InstId, ?MODULE, Options) of + case emqx_resource_pool:start(InstId, ?MODULE, [{log_tag, "eldap_info"} | Options]) of ok -> emqx_ldap_bind_worker:on_start( InstId, Config, - Options, + [{log_tag, "eldap_bind_info"} | Options], prepare_template(Config, #{pool_name => InstId}) ); {error, Reason} -> @@ -193,7 +195,15 @@ on_query(InstId, {query, Data, Attrs, Timeout}, State) -> on_query(InstId, {bind, _DN, _Data} = Req, State) -> emqx_ldap_bind_worker:on_query(InstId, Req, State). -on_get_status(_InstId, #{pool_name := PoolName} = _State) -> +on_get_status(InstId, #{pool_name := PoolName} = State) -> + case get_status_with_poolname(PoolName) of + connected -> + emqx_ldap_bind_worker:on_get_status(InstId, State); + disconnected -> + disconnected + end. + +get_status_with_poolname(PoolName) -> case emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1) of true -> connected; @@ -209,7 +219,7 @@ do_get_status(Conn) -> %% if the server is down, the result is {error, ldap_closed} %% otherwise is {error, invalidDNSyntax/timeout} {error, ldap_closed} =/= - eldap:search(Conn, [{base, "checkalive"}, {filter, eldap:'approxMatch'("", "")}]). + eldap:search(Conn, [{base, "cn=checkalive"}, {filter, eldap:'approxMatch'("", "")}]). %% =================================================================== @@ -222,7 +232,8 @@ connect(Options) -> } = Conf = proplists:get_value(options, Options), OpenOpts = maps:to_list(maps:with([port, sslopts], Conf)), - case eldap:open([Host], [{log, fun log/3}, {timeout, RequestTimeout} | OpenOpts]) of + LogTag = proplists:get_value(log_tag, Options), + case eldap:open([Host], [{log, mk_log_func(LogTag)}, {timeout, RequestTimeout} | OpenOpts]) of {ok, Handle} = Ret -> %% TODO: teach `eldap` to accept 0-arity closures as passwords. case eldap:simple_bind(Handle, Username, emqx_secret:unwrap(Password)) of @@ -313,14 +324,21 @@ do_ldap_query( end. %% Note: the value of the `_Level` here always is 2 -log(_Level, Format, Args) -> - ?SLOG( - info, - #{ - msg => "eldap_info", - log => io_lib:format(Format, Args) - } - ). +mk_log_func(LogTag) -> + fun(_Level, Format, Args) -> + ?SLOG( + info, + #{ + msg => LogTag, + log => io_lib:format(Format, [redact_ldap_log(Arg) || Arg <- Args]) + } + ) + end. + +redact_ldap_log({'BindRequest', Version, Name, {simple, _}}) -> + {'BindRequest', Version, Name, {simple, ?REDACT_VAL}}; +redact_ldap_log(Arg) -> + Arg. prepare_template(Config, State) -> maps:fold(fun prepare_template/3, State, Config). diff --git a/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl b/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl index e19818893..d6259bf0e 100644 --- a/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl +++ b/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl @@ -24,7 +24,8 @@ -export([ on_start/4, on_stop/2, - on_query/3 + on_query/3, + on_get_status/2 ]). %% ecpool connect & reconnect @@ -103,6 +104,11 @@ on_query( {error, {unrecoverable_error, Reason}} end. +on_get_status(_InstId, #{bind_pool_name := PoolName}) -> + emqx_ldap:get_status_with_poolname(PoolName); +on_get_status(_InstId, _) -> + connected. + %% =================================================================== connect(Conf) -> From 239fd81661ed8febb98847f77e168306e7abd525 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 28 Feb 2024 21:58:03 +0800 Subject: [PATCH 21/25] chore: update change --- changes/ee/fix-12610.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ee/fix-12610.en.md diff --git a/changes/ee/fix-12610.en.md b/changes/ee/fix-12610.en.md new file mode 100644 index 000000000..e48020cce --- /dev/null +++ b/changes/ee/fix-12610.en.md @@ -0,0 +1 @@ +Fixed that the connection to the LDAP connector may be disconnected after a period of time. From 0670272188707d6d3330b9458472a492e43fe8c4 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 1 Mar 2024 10:32:57 +0800 Subject: [PATCH 22/25] fix: Revert "fix: redact all headers from logs" This reverts commit d8032f47ca1c184372a6f7368ab4187e4ebff939. --- apps/emqx_audit/test/emqx_audit_api_SUITE.erl | 2 +- apps/emqx_utils/src/emqx_utils.app.src | 2 +- apps/emqx_utils/src/emqx_utils.erl | 4 ---- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl index f1f4f2628..2f401e7a8 100644 --- a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl +++ b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl @@ -88,7 +88,7 @@ t_http_api(_) -> <<"method">> := <<"put">>, <<"body">> := #{<<"mqtt">> := #{<<"max_qos_allowed">> := 1}}, <<"bindings">> := _, - <<"headers">> := "******" + <<"headers">> := #{<<"authorization">> := <<"******">>} }, <<"http_status_code">> := 200, <<"operation_result">> := <<"success">>, diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index 8fdade473..766b25da6 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -2,7 +2,7 @@ {application, emqx_utils, [ {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! - {vsn, "5.0.16"}, + {vsn, "5.0.15"}, {modules, [ emqx_utils, emqx_utils_api, diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl index be9f99923..0eeef2e5e 100644 --- a/apps/emqx_utils/src/emqx_utils.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -717,9 +717,6 @@ is_sensitive_key(<<"jwt">>) -> true; is_sensitive_key(authorization) -> true; is_sensitive_key("authorization") -> true; is_sensitive_key(<<"authorization">>) -> true; -is_sensitive_key(headers) -> true; -is_sensitive_key("headers") -> true; -is_sensitive_key(<<"headers">>) -> true; is_sensitive_key(bind_password) -> true; is_sensitive_key("bind_password") -> true; is_sensitive_key(<<"bind_password">>) -> true; @@ -882,7 +879,6 @@ redact_test_() -> secret_key, secret_access_key, security_token, - headers, token, bind_password ], From 5a3a34cce795aa7ca6566e1a8abd0f815d7a3835 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 1 Mar 2024 12:30:26 +0800 Subject: [PATCH 23/25] fix(redact): enhanced the redact for sensitive headers --- .../src/emqx_bridge_http_connector.erl | 26 +- .../src/emqx_connector_resource.erl | 7 - apps/emqx_utils/src/emqx_utils.erl | 235 +------------ apps/emqx_utils/src/emqx_utils_redact.erl | 312 ++++++++++++++++++ 4 files changed, 319 insertions(+), 261 deletions(-) create mode 100644 apps/emqx_utils/src/emqx_utils_redact.erl diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index f62fc9d3f..88449251c 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -48,7 +48,7 @@ ]). %% for other http-like connectors. --export([redact_request/1, is_sensitive_key/1]). +-export([redact_request/1]). -export([validate_method/1, join_paths/2]). @@ -851,25 +851,10 @@ maybe_retry({error, Reason}, Context, ReplyFunAndArgs) -> maybe_retry(Result, _Context, ReplyFunAndArgs) -> emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result). -%% The HOCON schema system may generate sensitive keys with this format -is_sensitive_key(Atom) when is_atom(Atom) -> - is_sensitive_key(erlang:atom_to_binary(Atom)); -is_sensitive_key(Bin) when is_binary(Bin), (size(Bin) =:= 19 orelse size(Bin) =:= 13) -> - %% We want to convert this to lowercase since the http header fields - %% are case insensitive, which means that a user of the Webhook bridge - %% can write this field name in many different ways. - case try_bin_to_lower(Bin) of - <<"authorization">> -> true; - <<"proxy-authorization">> -> true; - _ -> false - end; -is_sensitive_key(_) -> - false. - %% Function that will do a deep traversal of Data and remove sensitive %% information (i.e., passwords) redact(Data) -> - emqx_utils:redact(Data, fun is_sensitive_key/1). + emqx_utils:redact(Data). %% because the body may contain some sensitive data %% and at the same time the redact function will not scan the binary data @@ -893,13 +878,6 @@ redact_test_() -> ] }, [ - ?_assert(is_sensitive_key(<<"Authorization">>)), - ?_assert(is_sensitive_key(<<"AuthoriZation">>)), - ?_assert(is_sensitive_key('AuthoriZation')), - ?_assert(is_sensitive_key(<<"PrOxy-authoRizaTion">>)), - ?_assert(is_sensitive_key('PrOxy-authoRizaTion')), - ?_assertNot(is_sensitive_key(<<"Something">>)), - ?_assertNot(is_sensitive_key(89)), ?_assertNotEqual(TestData, redact(TestData)) ]. diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index 4d753d477..27ac63f1b 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -379,12 +379,5 @@ override_start_after_created(Config, Opts) -> set_no_buffer_workers(Opts) -> Opts#{spawn_buffer_workers => false}. -%% TODO: introduce a formal callback? -redact(Conf, Type) when - Type =:= http; - Type =:= <<"http">> --> - %% CE bridge - emqx_utils:redact(Conf, fun emqx_bridge_http_connector:is_sensitive_key/1); redact(Conf, _Type) -> emqx_utils:redact(Conf). diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl index 0eeef2e5e..fdb854a5e 100644 --- a/apps/emqx_utils/src/emqx_utils.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -684,146 +684,20 @@ try_to_existing_atom(Convert, Data, Encoding) -> _:Reason -> {error, Reason} end. -%% NOTE: keep alphabetical order -is_sensitive_key(aws_secret_access_key) -> true; -is_sensitive_key("aws_secret_access_key") -> true; -is_sensitive_key(<<"aws_secret_access_key">>) -> true; -is_sensitive_key(password) -> true; -is_sensitive_key("password") -> true; -is_sensitive_key(<<"password">>) -> true; -is_sensitive_key('proxy-authorization') -> true; -is_sensitive_key("proxy-authorization") -> true; -is_sensitive_key(<<"proxy-authorization">>) -> true; -is_sensitive_key(secret) -> true; -is_sensitive_key("secret") -> true; -is_sensitive_key(<<"secret">>) -> true; -is_sensitive_key(secret_access_key) -> true; -is_sensitive_key("secret_access_key") -> true; -is_sensitive_key(<<"secret_access_key">>) -> true; -is_sensitive_key(secret_key) -> true; -is_sensitive_key("secret_key") -> true; -is_sensitive_key(<<"secret_key">>) -> true; -is_sensitive_key(security_token) -> true; -is_sensitive_key("security_token") -> true; -is_sensitive_key(<<"security_token">>) -> true; -is_sensitive_key(sp_private_key) -> true; -is_sensitive_key(<<"sp_private_key">>) -> true; -is_sensitive_key(token) -> true; -is_sensitive_key("token") -> true; -is_sensitive_key(<<"token">>) -> true; -is_sensitive_key(jwt) -> true; -is_sensitive_key("jwt") -> true; -is_sensitive_key(<<"jwt">>) -> true; -is_sensitive_key(authorization) -> true; -is_sensitive_key("authorization") -> true; -is_sensitive_key(<<"authorization">>) -> true; -is_sensitive_key(bind_password) -> true; -is_sensitive_key("bind_password") -> true; -is_sensitive_key(<<"bind_password">>) -> true; -is_sensitive_key(Key) -> is_authorization(Key). - redact(Term) -> - do_redact(Term, fun is_sensitive_key/1). + emqx_utils_redact:redact(Term). redact(Term, Checker) -> - do_redact(Term, fun(V) -> - is_sensitive_key(V) orelse Checker(V) - end). - -do_redact(L, Checker) when is_list(L) -> - lists:map(fun(E) -> do_redact(E, Checker) end, L); -do_redact(M, Checker) when is_map(M) -> - maps:map( - fun(K, V) -> - do_redact(K, V, Checker) - end, - M - ); -do_redact({Key, Value}, Checker) -> - case Checker(Key) of - true -> - {Key, redact_v(Value)}; - false -> - {do_redact(Key, Checker), do_redact(Value, Checker)} - end; -do_redact(T, Checker) when is_tuple(T) -> - Elements = erlang:tuple_to_list(T), - Redact = do_redact(Elements, Checker), - erlang:list_to_tuple(Redact); -do_redact(Any, _Checker) -> - Any. - -do_redact(K, V, Checker) -> - case Checker(K) of - true -> - redact_v(V); - false -> - do_redact(V, Checker) - end. - --define(REDACT_VAL, "******"). -redact_v(V) when is_binary(V) -> <>; -%% The HOCON schema system may generate sensitive values with this format -redact_v([{str, Bin}]) when is_binary(Bin) -> - [{str, <>}]; -redact_v(_V) -> - ?REDACT_VAL. + emqx_utils_redact:redact(Term, Checker). deobfuscate(NewConf, OldConf) -> - maps:fold( - fun(K, V, Acc) -> - case maps:find(K, OldConf) of - error -> - case is_redacted(K, V) of - %% don't put redacted value into new config - true -> Acc; - false -> Acc#{K => V} - end; - {ok, OldV} when is_map(V), is_map(OldV) -> - Acc#{K => deobfuscate(V, OldV)}; - {ok, OldV} -> - case is_redacted(K, V) of - true -> - Acc#{K => OldV}; - _ -> - Acc#{K => V} - end - end - end, - #{}, - NewConf - ). + emqx_utils_redact:deobfuscate(NewConf, OldConf). is_redacted(K, V) -> - do_is_redacted(K, V, fun is_sensitive_key/1). + emqx_utils_redact:is_redacted(K, V). is_redacted(K, V, Fun) -> - do_is_redacted(K, V, fun(E) -> - is_sensitive_key(E) orelse Fun(E) - end). - -do_is_redacted(K, ?REDACT_VAL, Fun) -> - Fun(K); -do_is_redacted(K, <>, Fun) -> - Fun(K); -do_is_redacted(K, WrappedFun, Fun) when is_function(WrappedFun, 0) -> - %% wrapped by `emqx_secret' or other module - do_is_redacted(K, WrappedFun(), Fun); -do_is_redacted(_K, _V, _Fun) -> - false. - -%% This is ugly, however, the authorization is case-insensitive, -%% the best way is to check chars one by one and quickly exit when any position is not equal, -%% but in Erlang, this may not perform well, so here only check the first one -is_authorization([Cap | _] = Key) when Cap == $a; Cap == $A -> - is_authorization2(Key); -is_authorization(<> = Key) when Cap == $a; Cap == $A -> - is_authorization2(erlang:binary_to_list(Key)); -is_authorization(_Any) -> - false. - -is_authorization2(Str) -> - "authorization" == string:to_lower(Str). + emqx_utils_redact:is_redacted(K, V, Fun). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -837,105 +711,6 @@ ipv6_probe_test() -> ok end. -redact_test_() -> - Case = fun(Type, KeyT) -> - Key = - case Type of - atom -> KeyT; - string -> erlang:atom_to_list(KeyT); - binary -> erlang:atom_to_binary(KeyT) - end, - - ?assert(is_sensitive_key(Key)), - - %% direct - ?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo})), - ?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo})), - ?assertEqual({Key, Key, Key}, redact({Key, Key, Key})), - ?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar})), - - %% 1 level nested - ?assertEqual([{Key, ?REDACT_VAL}], redact([{Key, foo}])), - ?assertEqual([#{Key => ?REDACT_VAL}], redact([#{Key => foo}])), - - %% 2 level nested - ?assertEqual(#{opts => [{Key, ?REDACT_VAL}]}, redact(#{opts => [{Key, foo}]})), - ?assertEqual(#{opts => #{Key => ?REDACT_VAL}}, redact(#{opts => #{Key => foo}})), - ?assertEqual({opts, [{Key, ?REDACT_VAL}]}, redact({opts, [{Key, foo}]})), - - %% 3 level nested - ?assertEqual([#{opts => [{Key, ?REDACT_VAL}]}], redact([#{opts => [{Key, foo}]}])), - ?assertEqual([{opts, [{Key, ?REDACT_VAL}]}], redact([{opts, [{Key, foo}]}])), - ?assertEqual([{opts, [#{Key => ?REDACT_VAL}]}], redact([{opts, [#{Key => foo}]}])) - end, - - Types = [atom, string, binary], - Keys = [ - authorization, - aws_secret_access_key, - password, - 'proxy-authorization', - secret, - secret_key, - secret_access_key, - security_token, - token, - bind_password - ], - [{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types]. - -redact2_test_() -> - Case = fun(Key, Checker) -> - ?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo}, Checker)), - ?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo}, Checker)), - ?assertEqual({Key, Key, Key}, redact({Key, Key, Key}, Checker)), - ?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar}, Checker)) - end, - - Checker = fun(E) -> E =:= passcode end, - - Keys = [secret, passcode], - [{case_name(atom, Key), fun() -> Case(Key, Checker) end} || Key <- Keys]. - -deobfuscate_test() -> - NewConf0 = #{foo => <<"bar0">>, password => <<"123456">>}, - ?assertEqual(NewConf0, deobfuscate(NewConf0, #{foo => <<"bar">>, password => <<"654321">>})), - - NewConf1 = #{foo => <<"bar1">>, password => <>}, - ?assertEqual( - #{foo => <<"bar1">>, password => <<"654321">>}, - deobfuscate(NewConf1, #{foo => <<"bar">>, password => <<"654321">>}) - ), - - %% Don't have password before and ignore to put redact_val into new config - NewConf2 = #{foo => <<"bar2">>, password => ?REDACT_VAL}, - ?assertEqual(#{foo => <<"bar2">>}, deobfuscate(NewConf2, #{foo => <<"bar">>})), - - %% Don't have password before and should allow put non-redact-val into new config - NewConf3 = #{foo => <<"bar3">>, password => <<"123456">>}, - ?assertEqual(NewConf3, deobfuscate(NewConf3, #{foo => <<"bar">>})), - ok. - -redact_is_authorization_test_() -> - Types = [string, binary], - Keys = ["auThorization", "Authorization", "authorizaTion"], - - Case = fun(Type, Key0) -> - Key = - case Type of - binary -> - erlang:list_to_binary(Key0); - _ -> - Key0 - end, - ?assert(is_sensitive_key(Key)) - end, - - [{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types]. - -case_name(Type, Key) -> - lists:concat([Type, "-", Key]). - -endif. pub_props_to_packet(Properties) -> diff --git a/apps/emqx_utils/src/emqx_utils_redact.erl b/apps/emqx_utils/src/emqx_utils_redact.erl new file mode 100644 index 000000000..698d631e9 --- /dev/null +++ b/apps/emqx_utils/src/emqx_utils_redact.erl @@ -0,0 +1,312 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_utils_redact). + +-export([redact/1, redact/2, is_redacted/2, is_redacted/3]). +-export([deobfuscate/2]). + +-define(REDACT_VAL, "******"). +-define(IS_KEY_HEADERS(K), K == headers; K == <<"headers">>; K == "headers"). + +%% NOTE: keep alphabetical order +is_sensitive_key(aws_secret_access_key) -> true; +is_sensitive_key("aws_secret_access_key") -> true; +is_sensitive_key(<<"aws_secret_access_key">>) -> true; +is_sensitive_key(password) -> true; +is_sensitive_key("password") -> true; +is_sensitive_key(<<"password">>) -> true; +is_sensitive_key(secret) -> true; +is_sensitive_key("secret") -> true; +is_sensitive_key(<<"secret">>) -> true; +is_sensitive_key(secret_access_key) -> true; +is_sensitive_key("secret_access_key") -> true; +is_sensitive_key(<<"secret_access_key">>) -> true; +is_sensitive_key(secret_key) -> true; +is_sensitive_key("secret_key") -> true; +is_sensitive_key(<<"secret_key">>) -> true; +is_sensitive_key(security_token) -> true; +is_sensitive_key("security_token") -> true; +is_sensitive_key(<<"security_token">>) -> true; +is_sensitive_key(sp_private_key) -> true; +is_sensitive_key(<<"sp_private_key">>) -> true; +is_sensitive_key(token) -> true; +is_sensitive_key("token") -> true; +is_sensitive_key(<<"token">>) -> true; +is_sensitive_key(jwt) -> true; +is_sensitive_key("jwt") -> true; +is_sensitive_key(<<"jwt">>) -> true; +is_sensitive_key(bind_password) -> true; +is_sensitive_key("bind_password") -> true; +is_sensitive_key(<<"bind_password">>) -> true; +is_sensitive_key(_) -> false. + +redact(Term) -> + do_redact(Term, fun is_sensitive_key/1). + +redact(Term, Checker) -> + do_redact(Term, fun(V) -> + is_sensitive_key(V) orelse Checker(V) + end). + +do_redact(L, Checker) when is_list(L) -> + lists:map(fun(E) -> do_redact(E, Checker) end, L); +do_redact(M, Checker) when is_map(M) -> + maps:map( + fun(K, V) -> + do_redact(K, V, Checker) + end, + M + ); +do_redact({Headers, Value}, _Checker) when ?IS_KEY_HEADERS(Headers) -> + {Headers, do_redact_headers(Value)}; +do_redact({Key, Value}, Checker) -> + case Checker(Key) of + true -> + {Key, redact_v(Value)}; + false -> + {do_redact(Key, Checker), do_redact(Value, Checker)} + end; +do_redact(T, Checker) when is_tuple(T) -> + Elements = erlang:tuple_to_list(T), + Redact = do_redact(Elements, Checker), + erlang:list_to_tuple(Redact); +do_redact(Any, _Checker) -> + Any. + +do_redact(Headers, V, _Checker) when ?IS_KEY_HEADERS(Headers) -> + do_redact_headers(V); +do_redact(K, V, Checker) -> + case Checker(K) of + true -> + redact_v(V); + false -> + do_redact(V, Checker) + end. + +do_redact_headers(List) when is_list(List) -> + lists:map( + fun + ({K, V} = Pair) -> + case check_is_sensitive_header(K) of + true -> + {K, redact_v(V)}; + _ -> + Pair + end; + (Any) -> + Any + end, + List + ); +do_redact_headers(Map) when is_map(Map) -> + maps:map( + fun(K, V) -> + case check_is_sensitive_header(K) of + true -> + redact_v(V); + _ -> + V + end + end, + Map + ); +do_redact_headers(Value) -> + Value. + +check_is_sensitive_header(Key) -> + Key1 = emqx_utils_conv:str(Key), + is_sensitive_header(string:lowercase(Key1)). + +is_sensitive_header("authorization") -> + true; +is_sensitive_header("proxy-authorization") -> + true; +is_sensitive_header(_Any) -> + false. + +redact_v(V) when is_binary(V) -> <>; +%% The HOCON schema system may generate sensitive values with this format +redact_v([{str, Bin}]) when is_binary(Bin) -> + [{str, <>}]; +redact_v(_V) -> + ?REDACT_VAL. + +deobfuscate(NewConf, OldConf) -> + maps:fold( + fun(K, V, Acc) -> + case maps:find(K, OldConf) of + error -> + case is_redacted(K, V) of + %% don't put redacted value into new config + true -> Acc; + false -> Acc#{K => V} + end; + {ok, OldV} when is_map(V), is_map(OldV) -> + Acc#{K => deobfuscate(V, OldV)}; + {ok, OldV} -> + case is_redacted(K, V) of + true -> + Acc#{K => OldV}; + _ -> + Acc#{K => V} + end + end + end, + #{}, + NewConf + ). + +is_redacted(K, V) -> + do_is_redacted(K, V, fun is_sensitive_key/1). + +is_redacted(K, V, Fun) -> + do_is_redacted(K, V, fun(E) -> + is_sensitive_key(E) orelse Fun(E) + end). + +do_is_redacted(K, ?REDACT_VAL, Fun) -> + Fun(K); +do_is_redacted(K, <>, Fun) -> + Fun(K); +do_is_redacted(K, WrappedFun, Fun) when is_function(WrappedFun, 0) -> + %% wrapped by `emqx_secret' or other module + do_is_redacted(K, WrappedFun(), Fun); +do_is_redacted(_K, _V, _Fun) -> + false. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +redact_test_() -> + Case = fun(Type, KeyT) -> + Key = + case Type of + atom -> KeyT; + string -> erlang:atom_to_list(KeyT); + binary -> erlang:atom_to_binary(KeyT) + end, + + ?assert(is_sensitive_key(Key)), + + %% direct + ?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo})), + ?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo})), + ?assertEqual({Key, Key, Key}, redact({Key, Key, Key})), + ?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar})), + + %% 1 level nested + ?assertEqual([{Key, ?REDACT_VAL}], redact([{Key, foo}])), + ?assertEqual([#{Key => ?REDACT_VAL}], redact([#{Key => foo}])), + + %% 2 level nested + ?assertEqual(#{opts => [{Key, ?REDACT_VAL}]}, redact(#{opts => [{Key, foo}]})), + ?assertEqual(#{opts => #{Key => ?REDACT_VAL}}, redact(#{opts => #{Key => foo}})), + ?assertEqual({opts, [{Key, ?REDACT_VAL}]}, redact({opts, [{Key, foo}]})), + + %% 3 level nested + ?assertEqual([#{opts => [{Key, ?REDACT_VAL}]}], redact([#{opts => [{Key, foo}]}])), + ?assertEqual([{opts, [{Key, ?REDACT_VAL}]}], redact([{opts, [{Key, foo}]}])), + ?assertEqual([{opts, [#{Key => ?REDACT_VAL}]}], redact([{opts, [#{Key => foo}]}])) + end, + + Types = [atom, string, binary], + Keys = [ + aws_secret_access_key, + password, + secret, + secret_key, + secret_access_key, + security_token, + token, + bind_password + ], + [{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types]. + +redact2_test_() -> + Case = fun(Key, Checker) -> + ?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo}, Checker)), + ?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo}, Checker)), + ?assertEqual({Key, Key, Key}, redact({Key, Key, Key}, Checker)), + ?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar}, Checker)) + end, + + Checker = fun(E) -> E =:= passcode end, + + Keys = [secret, passcode], + [{case_name(atom, Key), fun() -> Case(Key, Checker) end} || Key <- Keys]. + +deobfuscate_test() -> + NewConf0 = #{foo => <<"bar0">>, password => <<"123456">>}, + ?assertEqual(NewConf0, deobfuscate(NewConf0, #{foo => <<"bar">>, password => <<"654321">>})), + + NewConf1 = #{foo => <<"bar1">>, password => <>}, + ?assertEqual( + #{foo => <<"bar1">>, password => <<"654321">>}, + deobfuscate(NewConf1, #{foo => <<"bar">>, password => <<"654321">>}) + ), + + %% Don't have password before and ignore to put redact_val into new config + NewConf2 = #{foo => <<"bar2">>, password => ?REDACT_VAL}, + ?assertEqual(#{foo => <<"bar2">>}, deobfuscate(NewConf2, #{foo => <<"bar">>})), + + %% Don't have password before and should allow put non-redact-val into new config + NewConf3 = #{foo => <<"bar3">>, password => <<"123456">>}, + ?assertEqual(NewConf3, deobfuscate(NewConf3, #{foo => <<"bar">>})), + ok. + +redact_header_test_() -> + Types = [string, binary, atom], + Keys = [ + "auThorization", + "Authorization", + "authorizaTion", + "proxy-authorizaTion", + "proXy-authoriZaTion" + ], + + Case = fun(Type, Key0) -> + Converter = + case Type of + binary -> + fun erlang:list_to_binary/1; + atom -> + fun erlang:list_to_atom/1; + _ -> + fun(Any) -> Any end + end, + + Name = Converter("headers"), + Key = Converter(Key0), + Value = Converter("value"), + Value1 = redact_v(Value), + ?assertMatch( + {Name, [{Key, Value1}]}, + redact({Name, [{Key, Value}]}) + ), + + ?assertMatch( + #{Name := #{Key := Value1}}, + redact(#{Name => #{Key => Value}}) + ) + end, + + [{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types]. + +case_name(Type, Key) -> + lists:concat([Type, "-", Key]). + +-endif. From 1c1c4e497d348d52844caf7ebc3c287bce7ad38b Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 1 Mar 2024 12:42:19 +0800 Subject: [PATCH 24/25] chore: bump version && update change --- apps/emqx_utils/src/emqx_utils.app.src | 2 +- changes/ce/fix-12620.en.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/ce/fix-12620.en.md diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index 766b25da6..8fdade473 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -2,7 +2,7 @@ {application, emqx_utils, [ {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! - {vsn, "5.0.15"}, + {vsn, "5.0.16"}, {modules, [ emqx_utils, emqx_utils_api, diff --git a/changes/ce/fix-12620.en.md b/changes/ce/fix-12620.en.md new file mode 100644 index 000000000..c54d66b41 --- /dev/null +++ b/changes/ce/fix-12620.en.md @@ -0,0 +1 @@ +Fixed the sensitive headers for HTTP connector may be printed in the `debug` level log. From 0a33f9d02701095d56f212d77a5ae32986cd8d6e Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 1 Mar 2024 12:49:31 +0800 Subject: [PATCH 25/25] fix(calendar): leap year time to unix timestamp --- .../test/emqx_rule_funcs_SUITE.erl | 22 +++++++++++++++++++ apps/emqx_utils/src/emqx_utils_calendar.erl | 11 +++++++++- changes/ce/fix-12632.en.md | 1 + 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 changes/ce/fix-12632.en.md diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 93b35fbc4..89e482c9c 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -1048,6 +1048,7 @@ t_parse_date_errors(_) -> ), %% Compatibility test + %% UTC+0 UnixTs = 1653561612, ?assertEqual( UnixTs, @@ -1062,6 +1063,27 @@ t_parse_date_errors(_) -> ?assertEqual( UnixTs, emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2022-05-26 10-40-12">>) + ), + + %% UTC+0 + UnixTsLeap0 = 1582986700, + ?assertEqual( + UnixTsLeap0, + emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2020-02-29 14:31:40">>) + ), + + %% UTC+0 + UnixTsLeap1 = 1709297071, + ?assertEqual( + UnixTsLeap1, + emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-03-01 12:44:31">>) + ), + + %% UTC+0 + UnixTsLeap2 = 1709535387, + ?assertEqual( + UnixTsLeap2, + emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-03-04 06:56:27">>) ). %%------------------------------------------------------------------------------ diff --git a/apps/emqx_utils/src/emqx_utils_calendar.erl b/apps/emqx_utils/src/emqx_utils_calendar.erl index 8f1e406bf..395d41314 100644 --- a/apps/emqx_utils/src/emqx_utils_calendar.erl +++ b/apps/emqx_utils/src/emqx_utils_calendar.erl @@ -478,7 +478,8 @@ do_parse(DateStr, Unit, Formatter) -> (year, V, Res) -> Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise); (month, V, Res) -> - Res + dm(V) * ?SECONDS_PER_DAY * Precise; + Dm = dym(maps:get(year, DateInfo, 0), V), + Res + Dm * ?SECONDS_PER_DAY * Precise; (day, V, Res) -> Res + (V * ?SECONDS_PER_DAY * Precise); (hour, V, Res) -> @@ -563,6 +564,14 @@ date_size(timezone) -> 5; date_size(timezone1) -> 6; date_size(timezone2) -> 9. +dym(Y, M) -> + case is_leap_year(Y) of + true when M > 2 -> + dm(M) + 1; + _ -> + dm(M) + end. + dm(1) -> 0; dm(2) -> 31; dm(3) -> 59; diff --git a/changes/ce/fix-12632.en.md b/changes/ce/fix-12632.en.md new file mode 100644 index 000000000..abfdef0df --- /dev/null +++ b/changes/ce/fix-12632.en.md @@ -0,0 +1 @@ +Fix incorrect results from rule SQL built-in function `date_to_unix_ts` after March on leap years.