fix(redact): enhanced the redact for sensitive headers
This commit is contained in:
parent
0670272188
commit
5a3a34cce7
|
@ -48,7 +48,7 @@
|
||||||
]).
|
]).
|
||||||
|
|
||||||
%% for other http-like connectors.
|
%% for other http-like connectors.
|
||||||
-export([redact_request/1, is_sensitive_key/1]).
|
-export([redact_request/1]).
|
||||||
|
|
||||||
-export([validate_method/1, join_paths/2]).
|
-export([validate_method/1, join_paths/2]).
|
||||||
|
|
||||||
|
@ -851,25 +851,10 @@ maybe_retry({error, Reason}, Context, ReplyFunAndArgs) ->
|
||||||
maybe_retry(Result, _Context, ReplyFunAndArgs) ->
|
maybe_retry(Result, _Context, ReplyFunAndArgs) ->
|
||||||
emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result).
|
emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result).
|
||||||
|
|
||||||
%% The HOCON schema system may generate sensitive keys with this format
|
|
||||||
is_sensitive_key(Atom) when is_atom(Atom) ->
|
|
||||||
is_sensitive_key(erlang:atom_to_binary(Atom));
|
|
||||||
is_sensitive_key(Bin) when is_binary(Bin), (size(Bin) =:= 19 orelse size(Bin) =:= 13) ->
|
|
||||||
%% We want to convert this to lowercase since the http header fields
|
|
||||||
%% are case insensitive, which means that a user of the Webhook bridge
|
|
||||||
%% can write this field name in many different ways.
|
|
||||||
case try_bin_to_lower(Bin) of
|
|
||||||
<<"authorization">> -> true;
|
|
||||||
<<"proxy-authorization">> -> true;
|
|
||||||
_ -> false
|
|
||||||
end;
|
|
||||||
is_sensitive_key(_) ->
|
|
||||||
false.
|
|
||||||
|
|
||||||
%% Function that will do a deep traversal of Data and remove sensitive
|
%% Function that will do a deep traversal of Data and remove sensitive
|
||||||
%% information (i.e., passwords)
|
%% information (i.e., passwords)
|
||||||
redact(Data) ->
|
redact(Data) ->
|
||||||
emqx_utils:redact(Data, fun is_sensitive_key/1).
|
emqx_utils:redact(Data).
|
||||||
|
|
||||||
%% because the body may contain some sensitive data
|
%% because the body may contain some sensitive data
|
||||||
%% and at the same time the redact function will not scan the binary data
|
%% and at the same time the redact function will not scan the binary data
|
||||||
|
@ -893,13 +878,6 @@ redact_test_() ->
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
[
|
[
|
||||||
?_assert(is_sensitive_key(<<"Authorization">>)),
|
|
||||||
?_assert(is_sensitive_key(<<"AuthoriZation">>)),
|
|
||||||
?_assert(is_sensitive_key('AuthoriZation')),
|
|
||||||
?_assert(is_sensitive_key(<<"PrOxy-authoRizaTion">>)),
|
|
||||||
?_assert(is_sensitive_key('PrOxy-authoRizaTion')),
|
|
||||||
?_assertNot(is_sensitive_key(<<"Something">>)),
|
|
||||||
?_assertNot(is_sensitive_key(89)),
|
|
||||||
?_assertNotEqual(TestData, redact(TestData))
|
?_assertNotEqual(TestData, redact(TestData))
|
||||||
].
|
].
|
||||||
|
|
||||||
|
|
|
@ -379,12 +379,5 @@ override_start_after_created(Config, Opts) ->
|
||||||
set_no_buffer_workers(Opts) ->
|
set_no_buffer_workers(Opts) ->
|
||||||
Opts#{spawn_buffer_workers => false}.
|
Opts#{spawn_buffer_workers => false}.
|
||||||
|
|
||||||
%% TODO: introduce a formal callback?
|
|
||||||
redact(Conf, Type) when
|
|
||||||
Type =:= http;
|
|
||||||
Type =:= <<"http">>
|
|
||||||
->
|
|
||||||
%% CE bridge
|
|
||||||
emqx_utils:redact(Conf, fun emqx_bridge_http_connector:is_sensitive_key/1);
|
|
||||||
redact(Conf, _Type) ->
|
redact(Conf, _Type) ->
|
||||||
emqx_utils:redact(Conf).
|
emqx_utils:redact(Conf).
|
||||||
|
|
|
@ -684,146 +684,20 @@ try_to_existing_atom(Convert, Data, Encoding) ->
|
||||||
_:Reason -> {error, Reason}
|
_:Reason -> {error, Reason}
|
||||||
end.
|
end.
|
||||||
|
|
||||||
%% NOTE: keep alphabetical order
|
|
||||||
is_sensitive_key(aws_secret_access_key) -> true;
|
|
||||||
is_sensitive_key("aws_secret_access_key") -> true;
|
|
||||||
is_sensitive_key(<<"aws_secret_access_key">>) -> true;
|
|
||||||
is_sensitive_key(password) -> true;
|
|
||||||
is_sensitive_key("password") -> true;
|
|
||||||
is_sensitive_key(<<"password">>) -> true;
|
|
||||||
is_sensitive_key('proxy-authorization') -> true;
|
|
||||||
is_sensitive_key("proxy-authorization") -> true;
|
|
||||||
is_sensitive_key(<<"proxy-authorization">>) -> true;
|
|
||||||
is_sensitive_key(secret) -> true;
|
|
||||||
is_sensitive_key("secret") -> true;
|
|
||||||
is_sensitive_key(<<"secret">>) -> true;
|
|
||||||
is_sensitive_key(secret_access_key) -> true;
|
|
||||||
is_sensitive_key("secret_access_key") -> true;
|
|
||||||
is_sensitive_key(<<"secret_access_key">>) -> true;
|
|
||||||
is_sensitive_key(secret_key) -> true;
|
|
||||||
is_sensitive_key("secret_key") -> true;
|
|
||||||
is_sensitive_key(<<"secret_key">>) -> true;
|
|
||||||
is_sensitive_key(security_token) -> true;
|
|
||||||
is_sensitive_key("security_token") -> true;
|
|
||||||
is_sensitive_key(<<"security_token">>) -> true;
|
|
||||||
is_sensitive_key(sp_private_key) -> true;
|
|
||||||
is_sensitive_key(<<"sp_private_key">>) -> true;
|
|
||||||
is_sensitive_key(token) -> true;
|
|
||||||
is_sensitive_key("token") -> true;
|
|
||||||
is_sensitive_key(<<"token">>) -> true;
|
|
||||||
is_sensitive_key(jwt) -> true;
|
|
||||||
is_sensitive_key("jwt") -> true;
|
|
||||||
is_sensitive_key(<<"jwt">>) -> true;
|
|
||||||
is_sensitive_key(authorization) -> true;
|
|
||||||
is_sensitive_key("authorization") -> true;
|
|
||||||
is_sensitive_key(<<"authorization">>) -> true;
|
|
||||||
is_sensitive_key(bind_password) -> true;
|
|
||||||
is_sensitive_key("bind_password") -> true;
|
|
||||||
is_sensitive_key(<<"bind_password">>) -> true;
|
|
||||||
is_sensitive_key(Key) -> is_authorization(Key).
|
|
||||||
|
|
||||||
redact(Term) ->
|
redact(Term) ->
|
||||||
do_redact(Term, fun is_sensitive_key/1).
|
emqx_utils_redact:redact(Term).
|
||||||
|
|
||||||
redact(Term, Checker) ->
|
redact(Term, Checker) ->
|
||||||
do_redact(Term, fun(V) ->
|
emqx_utils_redact:redact(Term, Checker).
|
||||||
is_sensitive_key(V) orelse Checker(V)
|
|
||||||
end).
|
|
||||||
|
|
||||||
do_redact(L, Checker) when is_list(L) ->
|
|
||||||
lists:map(fun(E) -> do_redact(E, Checker) end, L);
|
|
||||||
do_redact(M, Checker) when is_map(M) ->
|
|
||||||
maps:map(
|
|
||||||
fun(K, V) ->
|
|
||||||
do_redact(K, V, Checker)
|
|
||||||
end,
|
|
||||||
M
|
|
||||||
);
|
|
||||||
do_redact({Key, Value}, Checker) ->
|
|
||||||
case Checker(Key) of
|
|
||||||
true ->
|
|
||||||
{Key, redact_v(Value)};
|
|
||||||
false ->
|
|
||||||
{do_redact(Key, Checker), do_redact(Value, Checker)}
|
|
||||||
end;
|
|
||||||
do_redact(T, Checker) when is_tuple(T) ->
|
|
||||||
Elements = erlang:tuple_to_list(T),
|
|
||||||
Redact = do_redact(Elements, Checker),
|
|
||||||
erlang:list_to_tuple(Redact);
|
|
||||||
do_redact(Any, _Checker) ->
|
|
||||||
Any.
|
|
||||||
|
|
||||||
do_redact(K, V, Checker) ->
|
|
||||||
case Checker(K) of
|
|
||||||
true ->
|
|
||||||
redact_v(V);
|
|
||||||
false ->
|
|
||||||
do_redact(V, Checker)
|
|
||||||
end.
|
|
||||||
|
|
||||||
-define(REDACT_VAL, "******").
|
|
||||||
redact_v(V) when is_binary(V) -> <<?REDACT_VAL>>;
|
|
||||||
%% The HOCON schema system may generate sensitive values with this format
|
|
||||||
redact_v([{str, Bin}]) when is_binary(Bin) ->
|
|
||||||
[{str, <<?REDACT_VAL>>}];
|
|
||||||
redact_v(_V) ->
|
|
||||||
?REDACT_VAL.
|
|
||||||
|
|
||||||
deobfuscate(NewConf, OldConf) ->
|
deobfuscate(NewConf, OldConf) ->
|
||||||
maps:fold(
|
emqx_utils_redact:deobfuscate(NewConf, OldConf).
|
||||||
fun(K, V, Acc) ->
|
|
||||||
case maps:find(K, OldConf) of
|
|
||||||
error ->
|
|
||||||
case is_redacted(K, V) of
|
|
||||||
%% don't put redacted value into new config
|
|
||||||
true -> Acc;
|
|
||||||
false -> Acc#{K => V}
|
|
||||||
end;
|
|
||||||
{ok, OldV} when is_map(V), is_map(OldV) ->
|
|
||||||
Acc#{K => deobfuscate(V, OldV)};
|
|
||||||
{ok, OldV} ->
|
|
||||||
case is_redacted(K, V) of
|
|
||||||
true ->
|
|
||||||
Acc#{K => OldV};
|
|
||||||
_ ->
|
|
||||||
Acc#{K => V}
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end,
|
|
||||||
#{},
|
|
||||||
NewConf
|
|
||||||
).
|
|
||||||
|
|
||||||
is_redacted(K, V) ->
|
is_redacted(K, V) ->
|
||||||
do_is_redacted(K, V, fun is_sensitive_key/1).
|
emqx_utils_redact:is_redacted(K, V).
|
||||||
|
|
||||||
is_redacted(K, V, Fun) ->
|
is_redacted(K, V, Fun) ->
|
||||||
do_is_redacted(K, V, fun(E) ->
|
emqx_utils_redact:is_redacted(K, V, Fun).
|
||||||
is_sensitive_key(E) orelse Fun(E)
|
|
||||||
end).
|
|
||||||
|
|
||||||
do_is_redacted(K, ?REDACT_VAL, Fun) ->
|
|
||||||
Fun(K);
|
|
||||||
do_is_redacted(K, <<?REDACT_VAL>>, Fun) ->
|
|
||||||
Fun(K);
|
|
||||||
do_is_redacted(K, WrappedFun, Fun) when is_function(WrappedFun, 0) ->
|
|
||||||
%% wrapped by `emqx_secret' or other module
|
|
||||||
do_is_redacted(K, WrappedFun(), Fun);
|
|
||||||
do_is_redacted(_K, _V, _Fun) ->
|
|
||||||
false.
|
|
||||||
|
|
||||||
%% This is ugly, however, the authorization is case-insensitive,
|
|
||||||
%% the best way is to check chars one by one and quickly exit when any position is not equal,
|
|
||||||
%% but in Erlang, this may not perform well, so here only check the first one
|
|
||||||
is_authorization([Cap | _] = Key) when Cap == $a; Cap == $A ->
|
|
||||||
is_authorization2(Key);
|
|
||||||
is_authorization(<<Cap, _/binary>> = Key) when Cap == $a; Cap == $A ->
|
|
||||||
is_authorization2(erlang:binary_to_list(Key));
|
|
||||||
is_authorization(_Any) ->
|
|
||||||
false.
|
|
||||||
|
|
||||||
is_authorization2(Str) ->
|
|
||||||
"authorization" == string:to_lower(Str).
|
|
||||||
|
|
||||||
-ifdef(TEST).
|
-ifdef(TEST).
|
||||||
-include_lib("eunit/include/eunit.hrl").
|
-include_lib("eunit/include/eunit.hrl").
|
||||||
|
@ -837,105 +711,6 @@ ipv6_probe_test() ->
|
||||||
ok
|
ok
|
||||||
end.
|
end.
|
||||||
|
|
||||||
redact_test_() ->
|
|
||||||
Case = fun(Type, KeyT) ->
|
|
||||||
Key =
|
|
||||||
case Type of
|
|
||||||
atom -> KeyT;
|
|
||||||
string -> erlang:atom_to_list(KeyT);
|
|
||||||
binary -> erlang:atom_to_binary(KeyT)
|
|
||||||
end,
|
|
||||||
|
|
||||||
?assert(is_sensitive_key(Key)),
|
|
||||||
|
|
||||||
%% direct
|
|
||||||
?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo})),
|
|
||||||
?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo})),
|
|
||||||
?assertEqual({Key, Key, Key}, redact({Key, Key, Key})),
|
|
||||||
?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar})),
|
|
||||||
|
|
||||||
%% 1 level nested
|
|
||||||
?assertEqual([{Key, ?REDACT_VAL}], redact([{Key, foo}])),
|
|
||||||
?assertEqual([#{Key => ?REDACT_VAL}], redact([#{Key => foo}])),
|
|
||||||
|
|
||||||
%% 2 level nested
|
|
||||||
?assertEqual(#{opts => [{Key, ?REDACT_VAL}]}, redact(#{opts => [{Key, foo}]})),
|
|
||||||
?assertEqual(#{opts => #{Key => ?REDACT_VAL}}, redact(#{opts => #{Key => foo}})),
|
|
||||||
?assertEqual({opts, [{Key, ?REDACT_VAL}]}, redact({opts, [{Key, foo}]})),
|
|
||||||
|
|
||||||
%% 3 level nested
|
|
||||||
?assertEqual([#{opts => [{Key, ?REDACT_VAL}]}], redact([#{opts => [{Key, foo}]}])),
|
|
||||||
?assertEqual([{opts, [{Key, ?REDACT_VAL}]}], redact([{opts, [{Key, foo}]}])),
|
|
||||||
?assertEqual([{opts, [#{Key => ?REDACT_VAL}]}], redact([{opts, [#{Key => foo}]}]))
|
|
||||||
end,
|
|
||||||
|
|
||||||
Types = [atom, string, binary],
|
|
||||||
Keys = [
|
|
||||||
authorization,
|
|
||||||
aws_secret_access_key,
|
|
||||||
password,
|
|
||||||
'proxy-authorization',
|
|
||||||
secret,
|
|
||||||
secret_key,
|
|
||||||
secret_access_key,
|
|
||||||
security_token,
|
|
||||||
token,
|
|
||||||
bind_password
|
|
||||||
],
|
|
||||||
[{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types].
|
|
||||||
|
|
||||||
redact2_test_() ->
|
|
||||||
Case = fun(Key, Checker) ->
|
|
||||||
?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo}, Checker)),
|
|
||||||
?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo}, Checker)),
|
|
||||||
?assertEqual({Key, Key, Key}, redact({Key, Key, Key}, Checker)),
|
|
||||||
?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar}, Checker))
|
|
||||||
end,
|
|
||||||
|
|
||||||
Checker = fun(E) -> E =:= passcode end,
|
|
||||||
|
|
||||||
Keys = [secret, passcode],
|
|
||||||
[{case_name(atom, Key), fun() -> Case(Key, Checker) end} || Key <- Keys].
|
|
||||||
|
|
||||||
deobfuscate_test() ->
|
|
||||||
NewConf0 = #{foo => <<"bar0">>, password => <<"123456">>},
|
|
||||||
?assertEqual(NewConf0, deobfuscate(NewConf0, #{foo => <<"bar">>, password => <<"654321">>})),
|
|
||||||
|
|
||||||
NewConf1 = #{foo => <<"bar1">>, password => <<?REDACT_VAL>>},
|
|
||||||
?assertEqual(
|
|
||||||
#{foo => <<"bar1">>, password => <<"654321">>},
|
|
||||||
deobfuscate(NewConf1, #{foo => <<"bar">>, password => <<"654321">>})
|
|
||||||
),
|
|
||||||
|
|
||||||
%% Don't have password before and ignore to put redact_val into new config
|
|
||||||
NewConf2 = #{foo => <<"bar2">>, password => ?REDACT_VAL},
|
|
||||||
?assertEqual(#{foo => <<"bar2">>}, deobfuscate(NewConf2, #{foo => <<"bar">>})),
|
|
||||||
|
|
||||||
%% Don't have password before and should allow put non-redact-val into new config
|
|
||||||
NewConf3 = #{foo => <<"bar3">>, password => <<"123456">>},
|
|
||||||
?assertEqual(NewConf3, deobfuscate(NewConf3, #{foo => <<"bar">>})),
|
|
||||||
ok.
|
|
||||||
|
|
||||||
redact_is_authorization_test_() ->
|
|
||||||
Types = [string, binary],
|
|
||||||
Keys = ["auThorization", "Authorization", "authorizaTion"],
|
|
||||||
|
|
||||||
Case = fun(Type, Key0) ->
|
|
||||||
Key =
|
|
||||||
case Type of
|
|
||||||
binary ->
|
|
||||||
erlang:list_to_binary(Key0);
|
|
||||||
_ ->
|
|
||||||
Key0
|
|
||||||
end,
|
|
||||||
?assert(is_sensitive_key(Key))
|
|
||||||
end,
|
|
||||||
|
|
||||||
[{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types].
|
|
||||||
|
|
||||||
case_name(Type, Key) ->
|
|
||||||
lists:concat([Type, "-", Key]).
|
|
||||||
|
|
||||||
-endif.
|
-endif.
|
||||||
|
|
||||||
pub_props_to_packet(Properties) ->
|
pub_props_to_packet(Properties) ->
|
||||||
|
|
|
@ -0,0 +1,312 @@
|
||||||
|
%%--------------------------------------------------------------------
|
||||||
|
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||||
|
%%
|
||||||
|
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
%% you may not use this file except in compliance with the License.
|
||||||
|
%% You may obtain a copy of the License at
|
||||||
|
%%
|
||||||
|
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
%%
|
||||||
|
%% Unless required by applicable law or agreed to in writing, software
|
||||||
|
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
%% See the License for the specific language governing permissions and
|
||||||
|
%% limitations under the License.
|
||||||
|
%%--------------------------------------------------------------------
|
||||||
|
|
||||||
|
-module(emqx_utils_redact).
|
||||||
|
|
||||||
|
-export([redact/1, redact/2, is_redacted/2, is_redacted/3]).
|
||||||
|
-export([deobfuscate/2]).
|
||||||
|
|
||||||
|
-define(REDACT_VAL, "******").
|
||||||
|
-define(IS_KEY_HEADERS(K), K == headers; K == <<"headers">>; K == "headers").
|
||||||
|
|
||||||
|
%% NOTE: keep alphabetical order
|
||||||
|
is_sensitive_key(aws_secret_access_key) -> true;
|
||||||
|
is_sensitive_key("aws_secret_access_key") -> true;
|
||||||
|
is_sensitive_key(<<"aws_secret_access_key">>) -> true;
|
||||||
|
is_sensitive_key(password) -> true;
|
||||||
|
is_sensitive_key("password") -> true;
|
||||||
|
is_sensitive_key(<<"password">>) -> true;
|
||||||
|
is_sensitive_key(secret) -> true;
|
||||||
|
is_sensitive_key("secret") -> true;
|
||||||
|
is_sensitive_key(<<"secret">>) -> true;
|
||||||
|
is_sensitive_key(secret_access_key) -> true;
|
||||||
|
is_sensitive_key("secret_access_key") -> true;
|
||||||
|
is_sensitive_key(<<"secret_access_key">>) -> true;
|
||||||
|
is_sensitive_key(secret_key) -> true;
|
||||||
|
is_sensitive_key("secret_key") -> true;
|
||||||
|
is_sensitive_key(<<"secret_key">>) -> true;
|
||||||
|
is_sensitive_key(security_token) -> true;
|
||||||
|
is_sensitive_key("security_token") -> true;
|
||||||
|
is_sensitive_key(<<"security_token">>) -> true;
|
||||||
|
is_sensitive_key(sp_private_key) -> true;
|
||||||
|
is_sensitive_key(<<"sp_private_key">>) -> true;
|
||||||
|
is_sensitive_key(token) -> true;
|
||||||
|
is_sensitive_key("token") -> true;
|
||||||
|
is_sensitive_key(<<"token">>) -> true;
|
||||||
|
is_sensitive_key(jwt) -> true;
|
||||||
|
is_sensitive_key("jwt") -> true;
|
||||||
|
is_sensitive_key(<<"jwt">>) -> true;
|
||||||
|
is_sensitive_key(bind_password) -> true;
|
||||||
|
is_sensitive_key("bind_password") -> true;
|
||||||
|
is_sensitive_key(<<"bind_password">>) -> true;
|
||||||
|
is_sensitive_key(_) -> false.
|
||||||
|
|
||||||
|
redact(Term) ->
|
||||||
|
do_redact(Term, fun is_sensitive_key/1).
|
||||||
|
|
||||||
|
redact(Term, Checker) ->
|
||||||
|
do_redact(Term, fun(V) ->
|
||||||
|
is_sensitive_key(V) orelse Checker(V)
|
||||||
|
end).
|
||||||
|
|
||||||
|
do_redact(L, Checker) when is_list(L) ->
|
||||||
|
lists:map(fun(E) -> do_redact(E, Checker) end, L);
|
||||||
|
do_redact(M, Checker) when is_map(M) ->
|
||||||
|
maps:map(
|
||||||
|
fun(K, V) ->
|
||||||
|
do_redact(K, V, Checker)
|
||||||
|
end,
|
||||||
|
M
|
||||||
|
);
|
||||||
|
do_redact({Headers, Value}, _Checker) when ?IS_KEY_HEADERS(Headers) ->
|
||||||
|
{Headers, do_redact_headers(Value)};
|
||||||
|
do_redact({Key, Value}, Checker) ->
|
||||||
|
case Checker(Key) of
|
||||||
|
true ->
|
||||||
|
{Key, redact_v(Value)};
|
||||||
|
false ->
|
||||||
|
{do_redact(Key, Checker), do_redact(Value, Checker)}
|
||||||
|
end;
|
||||||
|
do_redact(T, Checker) when is_tuple(T) ->
|
||||||
|
Elements = erlang:tuple_to_list(T),
|
||||||
|
Redact = do_redact(Elements, Checker),
|
||||||
|
erlang:list_to_tuple(Redact);
|
||||||
|
do_redact(Any, _Checker) ->
|
||||||
|
Any.
|
||||||
|
|
||||||
|
do_redact(Headers, V, _Checker) when ?IS_KEY_HEADERS(Headers) ->
|
||||||
|
do_redact_headers(V);
|
||||||
|
do_redact(K, V, Checker) ->
|
||||||
|
case Checker(K) of
|
||||||
|
true ->
|
||||||
|
redact_v(V);
|
||||||
|
false ->
|
||||||
|
do_redact(V, Checker)
|
||||||
|
end.
|
||||||
|
|
||||||
|
do_redact_headers(List) when is_list(List) ->
|
||||||
|
lists:map(
|
||||||
|
fun
|
||||||
|
({K, V} = Pair) ->
|
||||||
|
case check_is_sensitive_header(K) of
|
||||||
|
true ->
|
||||||
|
{K, redact_v(V)};
|
||||||
|
_ ->
|
||||||
|
Pair
|
||||||
|
end;
|
||||||
|
(Any) ->
|
||||||
|
Any
|
||||||
|
end,
|
||||||
|
List
|
||||||
|
);
|
||||||
|
do_redact_headers(Map) when is_map(Map) ->
|
||||||
|
maps:map(
|
||||||
|
fun(K, V) ->
|
||||||
|
case check_is_sensitive_header(K) of
|
||||||
|
true ->
|
||||||
|
redact_v(V);
|
||||||
|
_ ->
|
||||||
|
V
|
||||||
|
end
|
||||||
|
end,
|
||||||
|
Map
|
||||||
|
);
|
||||||
|
do_redact_headers(Value) ->
|
||||||
|
Value.
|
||||||
|
|
||||||
|
check_is_sensitive_header(Key) ->
|
||||||
|
Key1 = emqx_utils_conv:str(Key),
|
||||||
|
is_sensitive_header(string:lowercase(Key1)).
|
||||||
|
|
||||||
|
is_sensitive_header("authorization") ->
|
||||||
|
true;
|
||||||
|
is_sensitive_header("proxy-authorization") ->
|
||||||
|
true;
|
||||||
|
is_sensitive_header(_Any) ->
|
||||||
|
false.
|
||||||
|
|
||||||
|
redact_v(V) when is_binary(V) -> <<?REDACT_VAL>>;
|
||||||
|
%% The HOCON schema system may generate sensitive values with this format
|
||||||
|
redact_v([{str, Bin}]) when is_binary(Bin) ->
|
||||||
|
[{str, <<?REDACT_VAL>>}];
|
||||||
|
redact_v(_V) ->
|
||||||
|
?REDACT_VAL.
|
||||||
|
|
||||||
|
deobfuscate(NewConf, OldConf) ->
|
||||||
|
maps:fold(
|
||||||
|
fun(K, V, Acc) ->
|
||||||
|
case maps:find(K, OldConf) of
|
||||||
|
error ->
|
||||||
|
case is_redacted(K, V) of
|
||||||
|
%% don't put redacted value into new config
|
||||||
|
true -> Acc;
|
||||||
|
false -> Acc#{K => V}
|
||||||
|
end;
|
||||||
|
{ok, OldV} when is_map(V), is_map(OldV) ->
|
||||||
|
Acc#{K => deobfuscate(V, OldV)};
|
||||||
|
{ok, OldV} ->
|
||||||
|
case is_redacted(K, V) of
|
||||||
|
true ->
|
||||||
|
Acc#{K => OldV};
|
||||||
|
_ ->
|
||||||
|
Acc#{K => V}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end,
|
||||||
|
#{},
|
||||||
|
NewConf
|
||||||
|
).
|
||||||
|
|
||||||
|
is_redacted(K, V) ->
|
||||||
|
do_is_redacted(K, V, fun is_sensitive_key/1).
|
||||||
|
|
||||||
|
is_redacted(K, V, Fun) ->
|
||||||
|
do_is_redacted(K, V, fun(E) ->
|
||||||
|
is_sensitive_key(E) orelse Fun(E)
|
||||||
|
end).
|
||||||
|
|
||||||
|
do_is_redacted(K, ?REDACT_VAL, Fun) ->
|
||||||
|
Fun(K);
|
||||||
|
do_is_redacted(K, <<?REDACT_VAL>>, Fun) ->
|
||||||
|
Fun(K);
|
||||||
|
do_is_redacted(K, WrappedFun, Fun) when is_function(WrappedFun, 0) ->
|
||||||
|
%% wrapped by `emqx_secret' or other module
|
||||||
|
do_is_redacted(K, WrappedFun(), Fun);
|
||||||
|
do_is_redacted(_K, _V, _Fun) ->
|
||||||
|
false.
|
||||||
|
|
||||||
|
-ifdef(TEST).
|
||||||
|
-include_lib("eunit/include/eunit.hrl").
|
||||||
|
|
||||||
|
redact_test_() ->
|
||||||
|
Case = fun(Type, KeyT) ->
|
||||||
|
Key =
|
||||||
|
case Type of
|
||||||
|
atom -> KeyT;
|
||||||
|
string -> erlang:atom_to_list(KeyT);
|
||||||
|
binary -> erlang:atom_to_binary(KeyT)
|
||||||
|
end,
|
||||||
|
|
||||||
|
?assert(is_sensitive_key(Key)),
|
||||||
|
|
||||||
|
%% direct
|
||||||
|
?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo})),
|
||||||
|
?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo})),
|
||||||
|
?assertEqual({Key, Key, Key}, redact({Key, Key, Key})),
|
||||||
|
?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar})),
|
||||||
|
|
||||||
|
%% 1 level nested
|
||||||
|
?assertEqual([{Key, ?REDACT_VAL}], redact([{Key, foo}])),
|
||||||
|
?assertEqual([#{Key => ?REDACT_VAL}], redact([#{Key => foo}])),
|
||||||
|
|
||||||
|
%% 2 level nested
|
||||||
|
?assertEqual(#{opts => [{Key, ?REDACT_VAL}]}, redact(#{opts => [{Key, foo}]})),
|
||||||
|
?assertEqual(#{opts => #{Key => ?REDACT_VAL}}, redact(#{opts => #{Key => foo}})),
|
||||||
|
?assertEqual({opts, [{Key, ?REDACT_VAL}]}, redact({opts, [{Key, foo}]})),
|
||||||
|
|
||||||
|
%% 3 level nested
|
||||||
|
?assertEqual([#{opts => [{Key, ?REDACT_VAL}]}], redact([#{opts => [{Key, foo}]}])),
|
||||||
|
?assertEqual([{opts, [{Key, ?REDACT_VAL}]}], redact([{opts, [{Key, foo}]}])),
|
||||||
|
?assertEqual([{opts, [#{Key => ?REDACT_VAL}]}], redact([{opts, [#{Key => foo}]}]))
|
||||||
|
end,
|
||||||
|
|
||||||
|
Types = [atom, string, binary],
|
||||||
|
Keys = [
|
||||||
|
aws_secret_access_key,
|
||||||
|
password,
|
||||||
|
secret,
|
||||||
|
secret_key,
|
||||||
|
secret_access_key,
|
||||||
|
security_token,
|
||||||
|
token,
|
||||||
|
bind_password
|
||||||
|
],
|
||||||
|
[{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types].
|
||||||
|
|
||||||
|
redact2_test_() ->
|
||||||
|
Case = fun(Key, Checker) ->
|
||||||
|
?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo}, Checker)),
|
||||||
|
?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo}, Checker)),
|
||||||
|
?assertEqual({Key, Key, Key}, redact({Key, Key, Key}, Checker)),
|
||||||
|
?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar}, Checker))
|
||||||
|
end,
|
||||||
|
|
||||||
|
Checker = fun(E) -> E =:= passcode end,
|
||||||
|
|
||||||
|
Keys = [secret, passcode],
|
||||||
|
[{case_name(atom, Key), fun() -> Case(Key, Checker) end} || Key <- Keys].
|
||||||
|
|
||||||
|
deobfuscate_test() ->
|
||||||
|
NewConf0 = #{foo => <<"bar0">>, password => <<"123456">>},
|
||||||
|
?assertEqual(NewConf0, deobfuscate(NewConf0, #{foo => <<"bar">>, password => <<"654321">>})),
|
||||||
|
|
||||||
|
NewConf1 = #{foo => <<"bar1">>, password => <<?REDACT_VAL>>},
|
||||||
|
?assertEqual(
|
||||||
|
#{foo => <<"bar1">>, password => <<"654321">>},
|
||||||
|
deobfuscate(NewConf1, #{foo => <<"bar">>, password => <<"654321">>})
|
||||||
|
),
|
||||||
|
|
||||||
|
%% Don't have password before and ignore to put redact_val into new config
|
||||||
|
NewConf2 = #{foo => <<"bar2">>, password => ?REDACT_VAL},
|
||||||
|
?assertEqual(#{foo => <<"bar2">>}, deobfuscate(NewConf2, #{foo => <<"bar">>})),
|
||||||
|
|
||||||
|
%% Don't have password before and should allow put non-redact-val into new config
|
||||||
|
NewConf3 = #{foo => <<"bar3">>, password => <<"123456">>},
|
||||||
|
?assertEqual(NewConf3, deobfuscate(NewConf3, #{foo => <<"bar">>})),
|
||||||
|
ok.
|
||||||
|
|
||||||
|
redact_header_test_() ->
|
||||||
|
Types = [string, binary, atom],
|
||||||
|
Keys = [
|
||||||
|
"auThorization",
|
||||||
|
"Authorization",
|
||||||
|
"authorizaTion",
|
||||||
|
"proxy-authorizaTion",
|
||||||
|
"proXy-authoriZaTion"
|
||||||
|
],
|
||||||
|
|
||||||
|
Case = fun(Type, Key0) ->
|
||||||
|
Converter =
|
||||||
|
case Type of
|
||||||
|
binary ->
|
||||||
|
fun erlang:list_to_binary/1;
|
||||||
|
atom ->
|
||||||
|
fun erlang:list_to_atom/1;
|
||||||
|
_ ->
|
||||||
|
fun(Any) -> Any end
|
||||||
|
end,
|
||||||
|
|
||||||
|
Name = Converter("headers"),
|
||||||
|
Key = Converter(Key0),
|
||||||
|
Value = Converter("value"),
|
||||||
|
Value1 = redact_v(Value),
|
||||||
|
?assertMatch(
|
||||||
|
{Name, [{Key, Value1}]},
|
||||||
|
redact({Name, [{Key, Value}]})
|
||||||
|
),
|
||||||
|
|
||||||
|
?assertMatch(
|
||||||
|
#{Name := #{Key := Value1}},
|
||||||
|
redact(#{Name => #{Key => Value}})
|
||||||
|
)
|
||||||
|
end,
|
||||||
|
|
||||||
|
[{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types].
|
||||||
|
|
||||||
|
case_name(Type, Key) ->
|
||||||
|
lists:concat([Type, "-", Key]).
|
||||||
|
|
||||||
|
-endif.
|
Loading…
Reference in New Issue