yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(redact): enhanced the redact for sensitive headers

This commit is contained in:
firest 2024-03-01 12:30:26 +08:00
parent 0670272188
commit 5a3a34cce7
4 changed files with 319 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl
View File

@ -48,7 +48,7 @@
]).
%% for other http-like connectors.
-export([redact_request/1, is_sensitive_key/1]).
-export([redact_request/1]).
-export([validate_method/1, join_paths/2]).
@ -851,25 +851,10 @@ maybe_retry({error, Reason}, Context, ReplyFunAndArgs) ->
maybe_retry(Result, _Context, ReplyFunAndArgs) ->
emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result).
%% The HOCON schema system may generate sensitive keys with this format
is_sensitive_key(Atom) when is_atom(Atom) ->
is_sensitive_key(erlang:atom_to_binary(Atom));
is_sensitive_key(Bin) when is_binary(Bin), (size(Bin) =:= 19 orelse size(Bin) =:= 13) ->
%% We want to convert this to lowercase since the http header fields
%% are case insensitive, which means that a user of the Webhook bridge
%% can write this field name in many different ways.
case try_bin_to_lower(Bin) of
<<"authorization">> -> true;
<<"proxy-authorization">> -> true;
_ -> false
end;
is_sensitive_key(_) ->
false.
%% Function that will do a deep traversal of Data and remove sensitive
%% information (i.e., passwords)
redact(Data) ->
emqx_utils:redact(Data, fun is_sensitive_key/1).
emqx_utils:redact(Data).
%% because the body may contain some sensitive data
%% and at the same time the redact function will not scan the binary data
@ -893,13 +878,6 @@ redact_test_() ->
]
},
[
?_assert(is_sensitive_key(<<"Authorization">>)),
?_assert(is_sensitive_key(<<"AuthoriZation">>)),
?_assert(is_sensitive_key('AuthoriZation')),
?_assert(is_sensitive_key(<<"PrOxy-authoRizaTion">>)),
?_assert(is_sensitive_key('PrOxy-authoRizaTion')),
?_assertNot(is_sensitive_key(<<"Something">>)),
?_assertNot(is_sensitive_key(89)),
?_assertNotEqual(TestData, redact(TestData))
].

7
apps/emqx_connector/src/emqx_connector_resource.erl
View File

@ -379,12 +379,5 @@ override_start_after_created(Config, Opts) ->
set_no_buffer_workers(Opts) ->
Opts#{spawn_buffer_workers => false}.
%% TODO: introduce a formal callback?
redact(Conf, Type) when
Type =:= http;
Type =:= <<"http">>
->
%% CE bridge
emqx_utils:redact(Conf, fun emqx_bridge_http_connector:is_sensitive_key/1);
redact(Conf, _Type) ->
emqx_utils:redact(Conf).

235
apps/emqx_utils/src/emqx_utils.erl
View File

@ -684,146 +684,20 @@ try_to_existing_atom(Convert, Data, Encoding) ->
_:Reason -> {error, Reason}
end.
%% NOTE: keep alphabetical order
is_sensitive_key(aws_secret_access_key) -> true;
is_sensitive_key("aws_secret_access_key") -> true;
is_sensitive_key(<<"aws_secret_access_key">>) -> true;
is_sensitive_key(password) -> true;
is_sensitive_key("password") -> true;
is_sensitive_key(<<"password">>) -> true;
is_sensitive_key('proxy-authorization') -> true;
is_sensitive_key("proxy-authorization") -> true;
is_sensitive_key(<<"proxy-authorization">>) -> true;
is_sensitive_key(secret) -> true;
is_sensitive_key("secret") -> true;
is_sensitive_key(<<"secret">>) -> true;
is_sensitive_key(secret_access_key) -> true;
is_sensitive_key("secret_access_key") -> true;
is_sensitive_key(<<"secret_access_key">>) -> true;
is_sensitive_key(secret_key) -> true;
is_sensitive_key("secret_key") -> true;
is_sensitive_key(<<"secret_key">>) -> true;
is_sensitive_key(security_token) -> true;
is_sensitive_key("security_token") -> true;
is_sensitive_key(<<"security_token">>) -> true;
is_sensitive_key(sp_private_key) -> true;
is_sensitive_key(<<"sp_private_key">>) -> true;
is_sensitive_key(token) -> true;
is_sensitive_key("token") -> true;
is_sensitive_key(<<"token">>) -> true;
is_sensitive_key(jwt) -> true;
is_sensitive_key("jwt") -> true;
is_sensitive_key(<<"jwt">>) -> true;
is_sensitive_key(authorization) -> true;
is_sensitive_key("authorization") -> true;
is_sensitive_key(<<"authorization">>) -> true;
is_sensitive_key(bind_password) -> true;
is_sensitive_key("bind_password") -> true;
is_sensitive_key(<<"bind_password">>) -> true;
is_sensitive_key(Key) -> is_authorization(Key).
redact(Term) ->
do_redact(Term, fun is_sensitive_key/1).
emqx_utils_redact:redact(Term).
redact(Term, Checker) ->
do_redact(Term, fun(V) ->
is_sensitive_key(V) orelse Checker(V)
end).
do_redact(L, Checker) when is_list(L) ->
lists:map(fun(E) -> do_redact(E, Checker) end, L);
do_redact(M, Checker) when is_map(M) ->
maps:map(
fun(K, V) ->
do_redact(K, V, Checker)
end,
M
);
do_redact({Key, Value}, Checker) ->
case Checker(Key) of
true ->
{Key, redact_v(Value)};
false ->
{do_redact(Key, Checker), do_redact(Value, Checker)}
end;
do_redact(T, Checker) when is_tuple(T) ->
Elements = erlang:tuple_to_list(T),
Redact = do_redact(Elements, Checker),
erlang:list_to_tuple(Redact);
do_redact(Any, _Checker) ->
Any.
do_redact(K, V, Checker) ->
case Checker(K) of
true ->
redact_v(V);
false ->
do_redact(V, Checker)
end.
-define(REDACT_VAL, "******").
redact_v(V) when is_binary(V) -> <<?REDACT_VAL>>;
%% The HOCON schema system may generate sensitive values with this format
redact_v([{str, Bin}]) when is_binary(Bin) ->
[{str, <<?REDACT_VAL>>}];
redact_v(_V) ->
?REDACT_VAL.
emqx_utils_redact:redact(Term, Checker).
deobfuscate(NewConf, OldConf) ->
maps:fold(
fun(K, V, Acc) ->
case maps:find(K, OldConf) of
error ->
case is_redacted(K, V) of
%% don't put redacted value into new config
true -> Acc;
false -> Acc#{K => V}
end;
{ok, OldV} when is_map(V), is_map(OldV) ->
Acc#{K => deobfuscate(V, OldV)};
{ok, OldV} ->
case is_redacted(K, V) of
true ->
Acc#{K => OldV};
_ ->
Acc#{K => V}
end
end
end,
#{},
NewConf
).
emqx_utils_redact:deobfuscate(NewConf, OldConf).
is_redacted(K, V) ->
do_is_redacted(K, V, fun is_sensitive_key/1).
emqx_utils_redact:is_redacted(K, V).
is_redacted(K, V, Fun) ->
do_is_redacted(K, V, fun(E) ->
is_sensitive_key(E) orelse Fun(E)
end).
do_is_redacted(K, ?REDACT_VAL, Fun) ->
Fun(K);
do_is_redacted(K, <<?REDACT_VAL>>, Fun) ->
Fun(K);
do_is_redacted(K, WrappedFun, Fun) when is_function(WrappedFun, 0) ->
%% wrapped by `emqx_secret' or other module
do_is_redacted(K, WrappedFun(), Fun);
do_is_redacted(_K, _V, _Fun) ->
false.
%% This is ugly, however, the authorization is case-insensitive,
%% the best way is to check chars one by one and quickly exit when any position is not equal,
%% but in Erlang, this may not perform well, so here only check the first one
is_authorization([Cap | _] = Key) when Cap == $a; Cap == $A ->
is_authorization2(Key);
is_authorization(<<Cap, _/binary>> = Key) when Cap == $a; Cap == $A ->
is_authorization2(erlang:binary_to_list(Key));
is_authorization(_Any) ->
false.
is_authorization2(Str) ->
"authorization" == string:to_lower(Str).
emqx_utils_redact:is_redacted(K, V, Fun).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
@ -837,105 +711,6 @@ ipv6_probe_test() ->
ok
end.
redact_test_() ->
Case = fun(Type, KeyT) ->
Key =
case Type of
atom -> KeyT;
string -> erlang:atom_to_list(KeyT);
binary -> erlang:atom_to_binary(KeyT)
end,
?assert(is_sensitive_key(Key)),
%% direct
?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo})),
?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo})),
?assertEqual({Key, Key, Key}, redact({Key, Key, Key})),
?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar})),
%% 1 level nested
?assertEqual([{Key, ?REDACT_VAL}], redact([{Key, foo}])),
?assertEqual([#{Key => ?REDACT_VAL}], redact([#{Key => foo}])),
%% 2 level nested
?assertEqual(#{opts => [{Key, ?REDACT_VAL}]}, redact(#{opts => [{Key, foo}]})),
?assertEqual(#{opts => #{Key => ?REDACT_VAL}}, redact(#{opts => #{Key => foo}})),
?assertEqual({opts, [{Key, ?REDACT_VAL}]}, redact({opts, [{Key, foo}]})),
%% 3 level nested
?assertEqual([#{opts => [{Key, ?REDACT_VAL}]}], redact([#{opts => [{Key, foo}]}])),
?assertEqual([{opts, [{Key, ?REDACT_VAL}]}], redact([{opts, [{Key, foo}]}])),
?assertEqual([{opts, [#{Key => ?REDACT_VAL}]}], redact([{opts, [#{Key => foo}]}]))
end,
Types = [atom, string, binary],
Keys = [
authorization,
aws_secret_access_key,
password,
'proxy-authorization',
secret,
secret_key,
secret_access_key,
security_token,
token,
bind_password
],
[{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types].
redact2_test_() ->
Case = fun(Key, Checker) ->
?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo}, Checker)),
?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo}, Checker)),
?assertEqual({Key, Key, Key}, redact({Key, Key, Key}, Checker)),
?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar}, Checker))
end,
Checker = fun(E) -> E =:= passcode end,
Keys = [secret, passcode],
[{case_name(atom, Key), fun() -> Case(Key, Checker) end} || Key <- Keys].
deobfuscate_test() ->
NewConf0 = #{foo => <<"bar0">>, password => <<"123456">>},
?assertEqual(NewConf0, deobfuscate(NewConf0, #{foo => <<"bar">>, password => <<"654321">>})),
NewConf1 = #{foo => <<"bar1">>, password => <<?REDACT_VAL>>},
?assertEqual(
#{foo => <<"bar1">>, password => <<"654321">>},
deobfuscate(NewConf1, #{foo => <<"bar">>, password => <<"654321">>})
),
%% Don't have password before and ignore to put redact_val into new config
NewConf2 = #{foo => <<"bar2">>, password => ?REDACT_VAL},
?assertEqual(#{foo => <<"bar2">>}, deobfuscate(NewConf2, #{foo => <<"bar">>})),
%% Don't have password before and should allow put non-redact-val into new config
NewConf3 = #{foo => <<"bar3">>, password => <<"123456">>},
?assertEqual(NewConf3, deobfuscate(NewConf3, #{foo => <<"bar">>})),
ok.
redact_is_authorization_test_() ->
Types = [string, binary],
Keys = ["auThorization", "Authorization", "authorizaTion"],
Case = fun(Type, Key0) ->
Key =
case Type of
binary ->
erlang:list_to_binary(Key0);
_ ->
Key0
end,
?assert(is_sensitive_key(Key))
end,
[{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types].
case_name(Type, Key) ->
lists:concat([Type, "-", Key]).
-endif.
pub_props_to_packet(Properties) ->

312
apps/emqx_utils/src/emqx_utils_redact.erl Normal file
View File

@ -0,0 +1,312 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_utils_redact).
-export([redact/1, redact/2, is_redacted/2, is_redacted/3]).
-export([deobfuscate/2]).
-define(REDACT_VAL, "******").
-define(IS_KEY_HEADERS(K), K == headers; K == <<"headers">>; K == "headers").
%% NOTE: keep alphabetical order
is_sensitive_key(aws_secret_access_key) -> true;
is_sensitive_key("aws_secret_access_key") -> true;
is_sensitive_key(<<"aws_secret_access_key">>) -> true;
is_sensitive_key(password) -> true;
is_sensitive_key("password") -> true;
is_sensitive_key(<<"password">>) -> true;
is_sensitive_key(secret) -> true;
is_sensitive_key("secret") -> true;
is_sensitive_key(<<"secret">>) -> true;
is_sensitive_key(secret_access_key) -> true;
is_sensitive_key("secret_access_key") -> true;
is_sensitive_key(<<"secret_access_key">>) -> true;
is_sensitive_key(secret_key) -> true;
is_sensitive_key("secret_key") -> true;
is_sensitive_key(<<"secret_key">>) -> true;
is_sensitive_key(security_token) -> true;
is_sensitive_key("security_token") -> true;
is_sensitive_key(<<"security_token">>) -> true;
is_sensitive_key(sp_private_key) -> true;
is_sensitive_key(<<"sp_private_key">>) -> true;
is_sensitive_key(token) -> true;
is_sensitive_key("token") -> true;
is_sensitive_key(<<"token">>) -> true;
is_sensitive_key(jwt) -> true;
is_sensitive_key("jwt") -> true;
is_sensitive_key(<<"jwt">>) -> true;
is_sensitive_key(bind_password) -> true;
is_sensitive_key("bind_password") -> true;
is_sensitive_key(<<"bind_password">>) -> true;
is_sensitive_key(_) -> false.
redact(Term) ->
do_redact(Term, fun is_sensitive_key/1).
redact(Term, Checker) ->
do_redact(Term, fun(V) ->
is_sensitive_key(V) orelse Checker(V)
end).
do_redact(L, Checker) when is_list(L) ->
lists:map(fun(E) -> do_redact(E, Checker) end, L);
do_redact(M, Checker) when is_map(M) ->
maps:map(
fun(K, V) ->
do_redact(K, V, Checker)
end,
M
);
do_redact({Headers, Value}, _Checker) when ?IS_KEY_HEADERS(Headers) ->
{Headers, do_redact_headers(Value)};
do_redact({Key, Value}, Checker) ->
case Checker(Key) of
true ->
{Key, redact_v(Value)};
false ->
{do_redact(Key, Checker), do_redact(Value, Checker)}
end;
do_redact(T, Checker) when is_tuple(T) ->
Elements = erlang:tuple_to_list(T),
Redact = do_redact(Elements, Checker),
erlang:list_to_tuple(Redact);
do_redact(Any, _Checker) ->
Any.
do_redact(Headers, V, _Checker) when ?IS_KEY_HEADERS(Headers) ->
do_redact_headers(V);
do_redact(K, V, Checker) ->
case Checker(K) of
true ->
redact_v(V);
false ->
do_redact(V, Checker)
end.
do_redact_headers(List) when is_list(List) ->
lists:map(
fun
({K, V} = Pair) ->
case check_is_sensitive_header(K) of
true ->
{K, redact_v(V)};
_ ->
Pair
end;
(Any) ->
Any
end,
List
);
do_redact_headers(Map) when is_map(Map) ->
maps:map(
fun(K, V) ->
case check_is_sensitive_header(K) of
true ->
redact_v(V);
_ ->
V
end
end,
Map
);
do_redact_headers(Value) ->
Value.
check_is_sensitive_header(Key) ->
Key1 = emqx_utils_conv:str(Key),
is_sensitive_header(string:lowercase(Key1)).
is_sensitive_header("authorization") ->
true;
is_sensitive_header("proxy-authorization") ->
true;
is_sensitive_header(_Any) ->
false.
redact_v(V) when is_binary(V) -> <<?REDACT_VAL>>;
%% The HOCON schema system may generate sensitive values with this format
redact_v([{str, Bin}]) when is_binary(Bin) ->
[{str, <<?REDACT_VAL>>}];
redact_v(_V) ->
?REDACT_VAL.
deobfuscate(NewConf, OldConf) ->
maps:fold(
fun(K, V, Acc) ->
case maps:find(K, OldConf) of
error ->
case is_redacted(K, V) of
%% don't put redacted value into new config
true -> Acc;
false -> Acc#{K => V}
end;
{ok, OldV} when is_map(V), is_map(OldV) ->
Acc#{K => deobfuscate(V, OldV)};
{ok, OldV} ->
case is_redacted(K, V) of
true ->
Acc#{K => OldV};
_ ->
Acc#{K => V}
end
end
end,
#{},
NewConf
).
is_redacted(K, V) ->
do_is_redacted(K, V, fun is_sensitive_key/1).
is_redacted(K, V, Fun) ->
do_is_redacted(K, V, fun(E) ->
is_sensitive_key(E) orelse Fun(E)
end).
do_is_redacted(K, ?REDACT_VAL, Fun) ->
Fun(K);
do_is_redacted(K, <<?REDACT_VAL>>, Fun) ->
Fun(K);
do_is_redacted(K, WrappedFun, Fun) when is_function(WrappedFun, 0) ->
%% wrapped by `emqx_secret' or other module
do_is_redacted(K, WrappedFun(), Fun);
do_is_redacted(_K, _V, _Fun) ->
false.
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
redact_test_() ->
Case = fun(Type, KeyT) ->
Key =
case Type of
atom -> KeyT;
string -> erlang:atom_to_list(KeyT);
binary -> erlang:atom_to_binary(KeyT)
end,
?assert(is_sensitive_key(Key)),
%% direct
?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo})),
?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo})),
?assertEqual({Key, Key, Key}, redact({Key, Key, Key})),
?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar})),
%% 1 level nested
?assertEqual([{Key, ?REDACT_VAL}], redact([{Key, foo}])),
?assertEqual([#{Key => ?REDACT_VAL}], redact([#{Key => foo}])),
%% 2 level nested
?assertEqual(#{opts => [{Key, ?REDACT_VAL}]}, redact(#{opts => [{Key, foo}]})),
?assertEqual(#{opts => #{Key => ?REDACT_VAL}}, redact(#{opts => #{Key => foo}})),
?assertEqual({opts, [{Key, ?REDACT_VAL}]}, redact({opts, [{Key, foo}]})),
%% 3 level nested
?assertEqual([#{opts => [{Key, ?REDACT_VAL}]}], redact([#{opts => [{Key, foo}]}])),
?assertEqual([{opts, [{Key, ?REDACT_VAL}]}], redact([{opts, [{Key, foo}]}])),
?assertEqual([{opts, [#{Key => ?REDACT_VAL}]}], redact([{opts, [#{Key => foo}]}]))
end,
Types = [atom, string, binary],
Keys = [
aws_secret_access_key,
password,
secret,
secret_key,
secret_access_key,
security_token,
token,
bind_password
],
[{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types].
redact2_test_() ->
Case = fun(Key, Checker) ->
?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo}, Checker)),
?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo}, Checker)),
?assertEqual({Key, Key, Key}, redact({Key, Key, Key}, Checker)),
?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar}, Checker))
end,
Checker = fun(E) -> E =:= passcode end,
Keys = [secret, passcode],
[{case_name(atom, Key), fun() -> Case(Key, Checker) end} || Key <- Keys].
deobfuscate_test() ->
NewConf0 = #{foo => <<"bar0">>, password => <<"123456">>},
?assertEqual(NewConf0, deobfuscate(NewConf0, #{foo => <<"bar">>, password => <<"654321">>})),
NewConf1 = #{foo => <<"bar1">>, password => <<?REDACT_VAL>>},
?assertEqual(
#{foo => <<"bar1">>, password => <<"654321">>},
deobfuscate(NewConf1, #{foo => <<"bar">>, password => <<"654321">>})
),
%% Don't have password before and ignore to put redact_val into new config
NewConf2 = #{foo => <<"bar2">>, password => ?REDACT_VAL},
?assertEqual(#{foo => <<"bar2">>}, deobfuscate(NewConf2, #{foo => <<"bar">>})),
%% Don't have password before and should allow put non-redact-val into new config
NewConf3 = #{foo => <<"bar3">>, password => <<"123456">>},
?assertEqual(NewConf3, deobfuscate(NewConf3, #{foo => <<"bar">>})),
ok.
redact_header_test_() ->
Types = [string, binary, atom],
Keys = [
"auThorization",
"Authorization",
"authorizaTion",
"proxy-authorizaTion",
"proXy-authoriZaTion"
],
Case = fun(Type, Key0) ->
Converter =
case Type of
binary ->
fun erlang:list_to_binary/1;
atom ->
fun erlang:list_to_atom/1;
_ ->
fun(Any) -> Any end
end,
Name = Converter("headers"),
Key = Converter(Key0),
Value = Converter("value"),
Value1 = redact_v(Value),
?assertMatch(
{Name, [{Key, Value1}]},
redact({Name, [{Key, Value}]})
),
?assertMatch(
#{Name := #{Key := Value1}},
redact(#{Name => #{Key => Value}})
)
end,
[{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types].
case_name(Type, Key) ->
lists:concat([Type, "-", Key]).
-endif.