From 0670272188707d6d3330b9458472a492e43fe8c4 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 1 Mar 2024 10:32:57 +0800 Subject: [PATCH 1/3] fix: Revert "fix: redact all headers from logs" This reverts commit d8032f47ca1c184372a6f7368ab4187e4ebff939. --- apps/emqx_audit/test/emqx_audit_api_SUITE.erl | 2 +- apps/emqx_utils/src/emqx_utils.app.src | 2 +- apps/emqx_utils/src/emqx_utils.erl | 4 ---- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl index f1f4f2628..2f401e7a8 100644 --- a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl +++ b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl @@ -88,7 +88,7 @@ t_http_api(_) -> <<"method">> := <<"put">>, <<"body">> := #{<<"mqtt">> := #{<<"max_qos_allowed">> := 1}}, <<"bindings">> := _, - <<"headers">> := "******" + <<"headers">> := #{<<"authorization">> := <<"******">>} }, <<"http_status_code">> := 200, <<"operation_result">> := <<"success">>, diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index 8fdade473..766b25da6 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -2,7 +2,7 @@ {application, emqx_utils, [ {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! - {vsn, "5.0.16"}, + {vsn, "5.0.15"}, {modules, [ emqx_utils, emqx_utils_api, diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl index be9f99923..0eeef2e5e 100644 --- a/apps/emqx_utils/src/emqx_utils.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -717,9 +717,6 @@ is_sensitive_key(<<"jwt">>) -> true; is_sensitive_key(authorization) -> true; is_sensitive_key("authorization") -> true; is_sensitive_key(<<"authorization">>) -> true; -is_sensitive_key(headers) -> true; -is_sensitive_key("headers") -> true; -is_sensitive_key(<<"headers">>) -> true; is_sensitive_key(bind_password) -> true; is_sensitive_key("bind_password") -> true; is_sensitive_key(<<"bind_password">>) -> true; @@ -882,7 +879,6 @@ redact_test_() -> secret_key, secret_access_key, security_token, - headers, token, bind_password ], From 5a3a34cce795aa7ca6566e1a8abd0f815d7a3835 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 1 Mar 2024 12:30:26 +0800 Subject: [PATCH 2/3] fix(redact): enhanced the redact for sensitive headers --- .../src/emqx_bridge_http_connector.erl | 26 +- .../src/emqx_connector_resource.erl | 7 - apps/emqx_utils/src/emqx_utils.erl | 235 +------------ apps/emqx_utils/src/emqx_utils_redact.erl | 312 ++++++++++++++++++ 4 files changed, 319 insertions(+), 261 deletions(-) create mode 100644 apps/emqx_utils/src/emqx_utils_redact.erl diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index f62fc9d3f..88449251c 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -48,7 +48,7 @@ ]). %% for other http-like connectors. --export([redact_request/1, is_sensitive_key/1]). +-export([redact_request/1]). -export([validate_method/1, join_paths/2]). @@ -851,25 +851,10 @@ maybe_retry({error, Reason}, Context, ReplyFunAndArgs) -> maybe_retry(Result, _Context, ReplyFunAndArgs) -> emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result). -%% The HOCON schema system may generate sensitive keys with this format -is_sensitive_key(Atom) when is_atom(Atom) -> - is_sensitive_key(erlang:atom_to_binary(Atom)); -is_sensitive_key(Bin) when is_binary(Bin), (size(Bin) =:= 19 orelse size(Bin) =:= 13) -> - %% We want to convert this to lowercase since the http header fields - %% are case insensitive, which means that a user of the Webhook bridge - %% can write this field name in many different ways. - case try_bin_to_lower(Bin) of - <<"authorization">> -> true; - <<"proxy-authorization">> -> true; - _ -> false - end; -is_sensitive_key(_) -> - false. - %% Function that will do a deep traversal of Data and remove sensitive %% information (i.e., passwords) redact(Data) -> - emqx_utils:redact(Data, fun is_sensitive_key/1). + emqx_utils:redact(Data). %% because the body may contain some sensitive data %% and at the same time the redact function will not scan the binary data @@ -893,13 +878,6 @@ redact_test_() -> ] }, [ - ?_assert(is_sensitive_key(<<"Authorization">>)), - ?_assert(is_sensitive_key(<<"AuthoriZation">>)), - ?_assert(is_sensitive_key('AuthoriZation')), - ?_assert(is_sensitive_key(<<"PrOxy-authoRizaTion">>)), - ?_assert(is_sensitive_key('PrOxy-authoRizaTion')), - ?_assertNot(is_sensitive_key(<<"Something">>)), - ?_assertNot(is_sensitive_key(89)), ?_assertNotEqual(TestData, redact(TestData)) ]. diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index 4d753d477..27ac63f1b 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -379,12 +379,5 @@ override_start_after_created(Config, Opts) -> set_no_buffer_workers(Opts) -> Opts#{spawn_buffer_workers => false}. -%% TODO: introduce a formal callback? -redact(Conf, Type) when - Type =:= http; - Type =:= <<"http">> --> - %% CE bridge - emqx_utils:redact(Conf, fun emqx_bridge_http_connector:is_sensitive_key/1); redact(Conf, _Type) -> emqx_utils:redact(Conf). diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl index 0eeef2e5e..fdb854a5e 100644 --- a/apps/emqx_utils/src/emqx_utils.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -684,146 +684,20 @@ try_to_existing_atom(Convert, Data, Encoding) -> _:Reason -> {error, Reason} end. -%% NOTE: keep alphabetical order -is_sensitive_key(aws_secret_access_key) -> true; -is_sensitive_key("aws_secret_access_key") -> true; -is_sensitive_key(<<"aws_secret_access_key">>) -> true; -is_sensitive_key(password) -> true; -is_sensitive_key("password") -> true; -is_sensitive_key(<<"password">>) -> true; -is_sensitive_key('proxy-authorization') -> true; -is_sensitive_key("proxy-authorization") -> true; -is_sensitive_key(<<"proxy-authorization">>) -> true; -is_sensitive_key(secret) -> true; -is_sensitive_key("secret") -> true; -is_sensitive_key(<<"secret">>) -> true; -is_sensitive_key(secret_access_key) -> true; -is_sensitive_key("secret_access_key") -> true; -is_sensitive_key(<<"secret_access_key">>) -> true; -is_sensitive_key(secret_key) -> true; -is_sensitive_key("secret_key") -> true; -is_sensitive_key(<<"secret_key">>) -> true; -is_sensitive_key(security_token) -> true; -is_sensitive_key("security_token") -> true; -is_sensitive_key(<<"security_token">>) -> true; -is_sensitive_key(sp_private_key) -> true; -is_sensitive_key(<<"sp_private_key">>) -> true; -is_sensitive_key(token) -> true; -is_sensitive_key("token") -> true; -is_sensitive_key(<<"token">>) -> true; -is_sensitive_key(jwt) -> true; -is_sensitive_key("jwt") -> true; -is_sensitive_key(<<"jwt">>) -> true; -is_sensitive_key(authorization) -> true; -is_sensitive_key("authorization") -> true; -is_sensitive_key(<<"authorization">>) -> true; -is_sensitive_key(bind_password) -> true; -is_sensitive_key("bind_password") -> true; -is_sensitive_key(<<"bind_password">>) -> true; -is_sensitive_key(Key) -> is_authorization(Key). - redact(Term) -> - do_redact(Term, fun is_sensitive_key/1). + emqx_utils_redact:redact(Term). redact(Term, Checker) -> - do_redact(Term, fun(V) -> - is_sensitive_key(V) orelse Checker(V) - end). - -do_redact(L, Checker) when is_list(L) -> - lists:map(fun(E) -> do_redact(E, Checker) end, L); -do_redact(M, Checker) when is_map(M) -> - maps:map( - fun(K, V) -> - do_redact(K, V, Checker) - end, - M - ); -do_redact({Key, Value}, Checker) -> - case Checker(Key) of - true -> - {Key, redact_v(Value)}; - false -> - {do_redact(Key, Checker), do_redact(Value, Checker)} - end; -do_redact(T, Checker) when is_tuple(T) -> - Elements = erlang:tuple_to_list(T), - Redact = do_redact(Elements, Checker), - erlang:list_to_tuple(Redact); -do_redact(Any, _Checker) -> - Any. - -do_redact(K, V, Checker) -> - case Checker(K) of - true -> - redact_v(V); - false -> - do_redact(V, Checker) - end. - --define(REDACT_VAL, "******"). -redact_v(V) when is_binary(V) -> <>; -%% The HOCON schema system may generate sensitive values with this format -redact_v([{str, Bin}]) when is_binary(Bin) -> - [{str, <>}]; -redact_v(_V) -> - ?REDACT_VAL. + emqx_utils_redact:redact(Term, Checker). deobfuscate(NewConf, OldConf) -> - maps:fold( - fun(K, V, Acc) -> - case maps:find(K, OldConf) of - error -> - case is_redacted(K, V) of - %% don't put redacted value into new config - true -> Acc; - false -> Acc#{K => V} - end; - {ok, OldV} when is_map(V), is_map(OldV) -> - Acc#{K => deobfuscate(V, OldV)}; - {ok, OldV} -> - case is_redacted(K, V) of - true -> - Acc#{K => OldV}; - _ -> - Acc#{K => V} - end - end - end, - #{}, - NewConf - ). + emqx_utils_redact:deobfuscate(NewConf, OldConf). is_redacted(K, V) -> - do_is_redacted(K, V, fun is_sensitive_key/1). + emqx_utils_redact:is_redacted(K, V). is_redacted(K, V, Fun) -> - do_is_redacted(K, V, fun(E) -> - is_sensitive_key(E) orelse Fun(E) - end). - -do_is_redacted(K, ?REDACT_VAL, Fun) -> - Fun(K); -do_is_redacted(K, <>, Fun) -> - Fun(K); -do_is_redacted(K, WrappedFun, Fun) when is_function(WrappedFun, 0) -> - %% wrapped by `emqx_secret' or other module - do_is_redacted(K, WrappedFun(), Fun); -do_is_redacted(_K, _V, _Fun) -> - false. - -%% This is ugly, however, the authorization is case-insensitive, -%% the best way is to check chars one by one and quickly exit when any position is not equal, -%% but in Erlang, this may not perform well, so here only check the first one -is_authorization([Cap | _] = Key) when Cap == $a; Cap == $A -> - is_authorization2(Key); -is_authorization(<> = Key) when Cap == $a; Cap == $A -> - is_authorization2(erlang:binary_to_list(Key)); -is_authorization(_Any) -> - false. - -is_authorization2(Str) -> - "authorization" == string:to_lower(Str). + emqx_utils_redact:is_redacted(K, V, Fun). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -837,105 +711,6 @@ ipv6_probe_test() -> ok end. -redact_test_() -> - Case = fun(Type, KeyT) -> - Key = - case Type of - atom -> KeyT; - string -> erlang:atom_to_list(KeyT); - binary -> erlang:atom_to_binary(KeyT) - end, - - ?assert(is_sensitive_key(Key)), - - %% direct - ?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo})), - ?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo})), - ?assertEqual({Key, Key, Key}, redact({Key, Key, Key})), - ?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar})), - - %% 1 level nested - ?assertEqual([{Key, ?REDACT_VAL}], redact([{Key, foo}])), - ?assertEqual([#{Key => ?REDACT_VAL}], redact([#{Key => foo}])), - - %% 2 level nested - ?assertEqual(#{opts => [{Key, ?REDACT_VAL}]}, redact(#{opts => [{Key, foo}]})), - ?assertEqual(#{opts => #{Key => ?REDACT_VAL}}, redact(#{opts => #{Key => foo}})), - ?assertEqual({opts, [{Key, ?REDACT_VAL}]}, redact({opts, [{Key, foo}]})), - - %% 3 level nested - ?assertEqual([#{opts => [{Key, ?REDACT_VAL}]}], redact([#{opts => [{Key, foo}]}])), - ?assertEqual([{opts, [{Key, ?REDACT_VAL}]}], redact([{opts, [{Key, foo}]}])), - ?assertEqual([{opts, [#{Key => ?REDACT_VAL}]}], redact([{opts, [#{Key => foo}]}])) - end, - - Types = [atom, string, binary], - Keys = [ - authorization, - aws_secret_access_key, - password, - 'proxy-authorization', - secret, - secret_key, - secret_access_key, - security_token, - token, - bind_password - ], - [{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types]. - -redact2_test_() -> - Case = fun(Key, Checker) -> - ?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo}, Checker)), - ?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo}, Checker)), - ?assertEqual({Key, Key, Key}, redact({Key, Key, Key}, Checker)), - ?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar}, Checker)) - end, - - Checker = fun(E) -> E =:= passcode end, - - Keys = [secret, passcode], - [{case_name(atom, Key), fun() -> Case(Key, Checker) end} || Key <- Keys]. - -deobfuscate_test() -> - NewConf0 = #{foo => <<"bar0">>, password => <<"123456">>}, - ?assertEqual(NewConf0, deobfuscate(NewConf0, #{foo => <<"bar">>, password => <<"654321">>})), - - NewConf1 = #{foo => <<"bar1">>, password => <>}, - ?assertEqual( - #{foo => <<"bar1">>, password => <<"654321">>}, - deobfuscate(NewConf1, #{foo => <<"bar">>, password => <<"654321">>}) - ), - - %% Don't have password before and ignore to put redact_val into new config - NewConf2 = #{foo => <<"bar2">>, password => ?REDACT_VAL}, - ?assertEqual(#{foo => <<"bar2">>}, deobfuscate(NewConf2, #{foo => <<"bar">>})), - - %% Don't have password before and should allow put non-redact-val into new config - NewConf3 = #{foo => <<"bar3">>, password => <<"123456">>}, - ?assertEqual(NewConf3, deobfuscate(NewConf3, #{foo => <<"bar">>})), - ok. - -redact_is_authorization_test_() -> - Types = [string, binary], - Keys = ["auThorization", "Authorization", "authorizaTion"], - - Case = fun(Type, Key0) -> - Key = - case Type of - binary -> - erlang:list_to_binary(Key0); - _ -> - Key0 - end, - ?assert(is_sensitive_key(Key)) - end, - - [{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types]. - -case_name(Type, Key) -> - lists:concat([Type, "-", Key]). - -endif. pub_props_to_packet(Properties) -> diff --git a/apps/emqx_utils/src/emqx_utils_redact.erl b/apps/emqx_utils/src/emqx_utils_redact.erl new file mode 100644 index 000000000..698d631e9 --- /dev/null +++ b/apps/emqx_utils/src/emqx_utils_redact.erl @@ -0,0 +1,312 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_utils_redact). + +-export([redact/1, redact/2, is_redacted/2, is_redacted/3]). +-export([deobfuscate/2]). + +-define(REDACT_VAL, "******"). +-define(IS_KEY_HEADERS(K), K == headers; K == <<"headers">>; K == "headers"). + +%% NOTE: keep alphabetical order +is_sensitive_key(aws_secret_access_key) -> true; +is_sensitive_key("aws_secret_access_key") -> true; +is_sensitive_key(<<"aws_secret_access_key">>) -> true; +is_sensitive_key(password) -> true; +is_sensitive_key("password") -> true; +is_sensitive_key(<<"password">>) -> true; +is_sensitive_key(secret) -> true; +is_sensitive_key("secret") -> true; +is_sensitive_key(<<"secret">>) -> true; +is_sensitive_key(secret_access_key) -> true; +is_sensitive_key("secret_access_key") -> true; +is_sensitive_key(<<"secret_access_key">>) -> true; +is_sensitive_key(secret_key) -> true; +is_sensitive_key("secret_key") -> true; +is_sensitive_key(<<"secret_key">>) -> true; +is_sensitive_key(security_token) -> true; +is_sensitive_key("security_token") -> true; +is_sensitive_key(<<"security_token">>) -> true; +is_sensitive_key(sp_private_key) -> true; +is_sensitive_key(<<"sp_private_key">>) -> true; +is_sensitive_key(token) -> true; +is_sensitive_key("token") -> true; +is_sensitive_key(<<"token">>) -> true; +is_sensitive_key(jwt) -> true; +is_sensitive_key("jwt") -> true; +is_sensitive_key(<<"jwt">>) -> true; +is_sensitive_key(bind_password) -> true; +is_sensitive_key("bind_password") -> true; +is_sensitive_key(<<"bind_password">>) -> true; +is_sensitive_key(_) -> false. + +redact(Term) -> + do_redact(Term, fun is_sensitive_key/1). + +redact(Term, Checker) -> + do_redact(Term, fun(V) -> + is_sensitive_key(V) orelse Checker(V) + end). + +do_redact(L, Checker) when is_list(L) -> + lists:map(fun(E) -> do_redact(E, Checker) end, L); +do_redact(M, Checker) when is_map(M) -> + maps:map( + fun(K, V) -> + do_redact(K, V, Checker) + end, + M + ); +do_redact({Headers, Value}, _Checker) when ?IS_KEY_HEADERS(Headers) -> + {Headers, do_redact_headers(Value)}; +do_redact({Key, Value}, Checker) -> + case Checker(Key) of + true -> + {Key, redact_v(Value)}; + false -> + {do_redact(Key, Checker), do_redact(Value, Checker)} + end; +do_redact(T, Checker) when is_tuple(T) -> + Elements = erlang:tuple_to_list(T), + Redact = do_redact(Elements, Checker), + erlang:list_to_tuple(Redact); +do_redact(Any, _Checker) -> + Any. + +do_redact(Headers, V, _Checker) when ?IS_KEY_HEADERS(Headers) -> + do_redact_headers(V); +do_redact(K, V, Checker) -> + case Checker(K) of + true -> + redact_v(V); + false -> + do_redact(V, Checker) + end. + +do_redact_headers(List) when is_list(List) -> + lists:map( + fun + ({K, V} = Pair) -> + case check_is_sensitive_header(K) of + true -> + {K, redact_v(V)}; + _ -> + Pair + end; + (Any) -> + Any + end, + List + ); +do_redact_headers(Map) when is_map(Map) -> + maps:map( + fun(K, V) -> + case check_is_sensitive_header(K) of + true -> + redact_v(V); + _ -> + V + end + end, + Map + ); +do_redact_headers(Value) -> + Value. + +check_is_sensitive_header(Key) -> + Key1 = emqx_utils_conv:str(Key), + is_sensitive_header(string:lowercase(Key1)). + +is_sensitive_header("authorization") -> + true; +is_sensitive_header("proxy-authorization") -> + true; +is_sensitive_header(_Any) -> + false. + +redact_v(V) when is_binary(V) -> <>; +%% The HOCON schema system may generate sensitive values with this format +redact_v([{str, Bin}]) when is_binary(Bin) -> + [{str, <>}]; +redact_v(_V) -> + ?REDACT_VAL. + +deobfuscate(NewConf, OldConf) -> + maps:fold( + fun(K, V, Acc) -> + case maps:find(K, OldConf) of + error -> + case is_redacted(K, V) of + %% don't put redacted value into new config + true -> Acc; + false -> Acc#{K => V} + end; + {ok, OldV} when is_map(V), is_map(OldV) -> + Acc#{K => deobfuscate(V, OldV)}; + {ok, OldV} -> + case is_redacted(K, V) of + true -> + Acc#{K => OldV}; + _ -> + Acc#{K => V} + end + end + end, + #{}, + NewConf + ). + +is_redacted(K, V) -> + do_is_redacted(K, V, fun is_sensitive_key/1). + +is_redacted(K, V, Fun) -> + do_is_redacted(K, V, fun(E) -> + is_sensitive_key(E) orelse Fun(E) + end). + +do_is_redacted(K, ?REDACT_VAL, Fun) -> + Fun(K); +do_is_redacted(K, <>, Fun) -> + Fun(K); +do_is_redacted(K, WrappedFun, Fun) when is_function(WrappedFun, 0) -> + %% wrapped by `emqx_secret' or other module + do_is_redacted(K, WrappedFun(), Fun); +do_is_redacted(_K, _V, _Fun) -> + false. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +redact_test_() -> + Case = fun(Type, KeyT) -> + Key = + case Type of + atom -> KeyT; + string -> erlang:atom_to_list(KeyT); + binary -> erlang:atom_to_binary(KeyT) + end, + + ?assert(is_sensitive_key(Key)), + + %% direct + ?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo})), + ?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo})), + ?assertEqual({Key, Key, Key}, redact({Key, Key, Key})), + ?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar})), + + %% 1 level nested + ?assertEqual([{Key, ?REDACT_VAL}], redact([{Key, foo}])), + ?assertEqual([#{Key => ?REDACT_VAL}], redact([#{Key => foo}])), + + %% 2 level nested + ?assertEqual(#{opts => [{Key, ?REDACT_VAL}]}, redact(#{opts => [{Key, foo}]})), + ?assertEqual(#{opts => #{Key => ?REDACT_VAL}}, redact(#{opts => #{Key => foo}})), + ?assertEqual({opts, [{Key, ?REDACT_VAL}]}, redact({opts, [{Key, foo}]})), + + %% 3 level nested + ?assertEqual([#{opts => [{Key, ?REDACT_VAL}]}], redact([#{opts => [{Key, foo}]}])), + ?assertEqual([{opts, [{Key, ?REDACT_VAL}]}], redact([{opts, [{Key, foo}]}])), + ?assertEqual([{opts, [#{Key => ?REDACT_VAL}]}], redact([{opts, [#{Key => foo}]}])) + end, + + Types = [atom, string, binary], + Keys = [ + aws_secret_access_key, + password, + secret, + secret_key, + secret_access_key, + security_token, + token, + bind_password + ], + [{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types]. + +redact2_test_() -> + Case = fun(Key, Checker) -> + ?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo}, Checker)), + ?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo}, Checker)), + ?assertEqual({Key, Key, Key}, redact({Key, Key, Key}, Checker)), + ?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar}, Checker)) + end, + + Checker = fun(E) -> E =:= passcode end, + + Keys = [secret, passcode], + [{case_name(atom, Key), fun() -> Case(Key, Checker) end} || Key <- Keys]. + +deobfuscate_test() -> + NewConf0 = #{foo => <<"bar0">>, password => <<"123456">>}, + ?assertEqual(NewConf0, deobfuscate(NewConf0, #{foo => <<"bar">>, password => <<"654321">>})), + + NewConf1 = #{foo => <<"bar1">>, password => <>}, + ?assertEqual( + #{foo => <<"bar1">>, password => <<"654321">>}, + deobfuscate(NewConf1, #{foo => <<"bar">>, password => <<"654321">>}) + ), + + %% Don't have password before and ignore to put redact_val into new config + NewConf2 = #{foo => <<"bar2">>, password => ?REDACT_VAL}, + ?assertEqual(#{foo => <<"bar2">>}, deobfuscate(NewConf2, #{foo => <<"bar">>})), + + %% Don't have password before and should allow put non-redact-val into new config + NewConf3 = #{foo => <<"bar3">>, password => <<"123456">>}, + ?assertEqual(NewConf3, deobfuscate(NewConf3, #{foo => <<"bar">>})), + ok. + +redact_header_test_() -> + Types = [string, binary, atom], + Keys = [ + "auThorization", + "Authorization", + "authorizaTion", + "proxy-authorizaTion", + "proXy-authoriZaTion" + ], + + Case = fun(Type, Key0) -> + Converter = + case Type of + binary -> + fun erlang:list_to_binary/1; + atom -> + fun erlang:list_to_atom/1; + _ -> + fun(Any) -> Any end + end, + + Name = Converter("headers"), + Key = Converter(Key0), + Value = Converter("value"), + Value1 = redact_v(Value), + ?assertMatch( + {Name, [{Key, Value1}]}, + redact({Name, [{Key, Value}]}) + ), + + ?assertMatch( + #{Name := #{Key := Value1}}, + redact(#{Name => #{Key => Value}}) + ) + end, + + [{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types]. + +case_name(Type, Key) -> + lists:concat([Type, "-", Key]). + +-endif. From 1c1c4e497d348d52844caf7ebc3c287bce7ad38b Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 1 Mar 2024 12:42:19 +0800 Subject: [PATCH 3/3] chore: bump version && update change --- apps/emqx_utils/src/emqx_utils.app.src | 2 +- changes/ce/fix-12620.en.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/ce/fix-12620.en.md diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index 766b25da6..8fdade473 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -2,7 +2,7 @@ {application, emqx_utils, [ {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! - {vsn, "5.0.15"}, + {vsn, "5.0.16"}, {modules, [ emqx_utils, emqx_utils_api, diff --git a/changes/ce/fix-12620.en.md b/changes/ce/fix-12620.en.md new file mode 100644 index 000000000..c54d66b41 --- /dev/null +++ b/changes/ce/fix-12620.en.md @@ -0,0 +1 @@ +Fixed the sensitive headers for HTTP connector may be printed in the `debug` level log.