yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #12219 from zhongwencool/ft-deobfuscate-keys 

fix: FT deobfuscate secret key
This commit is contained in:
Zaiming (Stone) Shi 2023-12-22 13:58:08 +01:00 committed by GitHub
parent e9d453405a a16bce8c24
commit 157dc12047
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 126 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/emqx/src/emqx.app.src
View File

@ -2,7 +2,7 @@
{application, emqx, [
{id, "emqx"},
{description, "EMQX Core"},
{vsn, "5.1.15"},
{vsn, "5.1.16"},
{modules, []},
{registered, []},
{applications, [

2
apps/emqx_auth/src/emqx_auth.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_auth, [
{description, "EMQX Authentication and authorization"},
{vsn, "0.1.29"},
{vsn, "0.2.0"},
{modules, []},
{registered, [emqx_auth_sup]},
{applications, [

2
apps/emqx_auth_http/src/emqx_auth_http.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_auth_http, [
{description, "EMQX External HTTP API Authentication and Authorization"},
{vsn, "0.1.2"},
{vsn, "0.1.3"},
{registered, []},
{mod, {emqx_auth_http_app, []}},
{applications, [

2
apps/emqx_bridge/src/emqx_bridge.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_bridge, [
{description, "EMQX bridges"},
{vsn, "0.1.31"},
{vsn, "0.1.32"},
{registered, [emqx_bridge_sup]},
{mod, {emqx_bridge_app, []}},
{applications, [

27
apps/emqx_bridge/src/emqx_bridge_api.erl
View File

@ -495,7 +495,7 @@ schema("/bridges_probe") ->
case emqx_bridge:lookup(BridgeType, BridgeName) of
{ok, #{raw_config := RawConf}} ->
%% TODO will the maybe_upgrade step done by emqx_bridge:lookup cause any problems
Conf = deobfuscate(Conf1, RawConf),
Conf = emqx_utils:deobfuscate(Conf1, RawConf),
update_bridge(BridgeType, BridgeName, Conf);
{error, not_found} ->
?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
@ -587,9 +587,9 @@ maybe_deobfuscate_bridge_probe(#{<<"type">> := BridgeType0, <<"name">> := Bridge
BridgeType = upgrade_type(BridgeType0),
case emqx_bridge:lookup(BridgeType, BridgeName) of
{ok, #{raw_config := RawConf}} ->
%% TODO check if RawConf optained above is compatible with the commented out code below
%% TODO check if RawConf obtained above is compatible with the commented out code below
%% RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}),
deobfuscate(Params, RawConf);
emqx_utils:deobfuscate(Params, RawConf);
_ ->
%% A bridge may be probed before it's created, so not finding it here is fine
Params
@ -1123,27 +1123,6 @@ supported_versions(_Call) -> [1, 2, 3, 4, 5].
redact(Term) ->
emqx_utils:redact(Term).
deobfuscate(NewConf, OldConf) ->
maps:fold(
fun(K, V, Acc) ->
case maps:find(K, OldConf) of
error ->
Acc#{K => V};
{ok, OldV} when is_map(V), is_map(OldV) ->
Acc#{K => deobfuscate(V, OldV)};
{ok, OldV} ->
case emqx_utils:is_redacted(K, V) of
true ->
Acc#{K => OldV};
_ ->
Acc#{K => V}
end
end
end,
#{},
NewConf
).
map_to_json(M0) ->
%% When dealing with Hocon validation errors, `value' might contain non-serializable
%% values (e.g.: user_lookup_fun), so we try again without that key if serialization

27
apps/emqx_bridge/src/emqx_bridge_v2_api.erl
View File

@ -423,7 +423,7 @@ schema("/action_types") ->
case emqx_bridge_v2:lookup(BridgeType, BridgeName) of
{ok, _} ->
RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}),
Conf = deobfuscate(Conf1, RawConf),
Conf = emqx_utils:deobfuscate(Conf1, RawConf),
update_bridge(BridgeType, BridgeName, Conf);
{error, not_found} ->
?BRIDGE_NOT_FOUND(BridgeType, BridgeName)
@ -554,9 +554,9 @@ schema("/action_types") ->
maybe_deobfuscate_bridge_probe(#{<<"type">> := ActionType, <<"name">> := BridgeName} = Params) ->
case emqx_bridge_v2:lookup(ActionType, BridgeName) of
{ok, #{raw_config := RawConf}} ->
%% TODO check if RawConf optained above is compatible with the commented out code below
%% TODO check if RawConf obtained above is compatible with the commented out code below
%% RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}),
deobfuscate(Params, RawConf);
emqx_utils:deobfuscate(Params, RawConf);
_ ->
%% A bridge may be probed before it's created, so not finding it here is fine
Params
@ -586,27 +586,6 @@ is_ok(ResL) ->
ErrL -> hd(ErrL)
end.
deobfuscate(NewConf, OldConf) ->
maps:fold(
fun(K, V, Acc) ->
case maps:find(K, OldConf) of
error ->
Acc#{K => V};
{ok, OldV} when is_map(V), is_map(OldV) ->
Acc#{K => deobfuscate(V, OldV)};
{ok, OldV} ->
case emqx_utils:is_redacted(K, V) of
true ->
Acc#{K => OldV};
_ ->
Acc#{K => V}
end
end
end,
#{},
NewConf
).
%% bridge helpers
lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) ->
Nodes = mria:running_nodes(),

2
apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_gcp_pubsub, [
{description, "EMQX Enterprise GCP Pub/Sub Bridge"},
{vsn, "0.1.11"},
{vsn, "0.2.0"},
{registered, []},
{applications, [
kernel,

2
apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_greptimedb, [
{description, "EMQX GreptimeDB Bridge"},
{vsn, "0.1.5"},
{vsn, "0.1.6"},
{registered, []},
{applications, [
kernel,

2
apps/emqx_bridge_http/src/emqx_bridge_http.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_http, [
{description, "EMQX HTTP Bridge and Connector Application"},
{vsn, "0.1.6"},
{vsn, "0.2.0"},
{registered, []},
{applications, [kernel, stdlib, emqx_connector, emqx_resource, ehttpc]},
{env, [{emqx_action_info_modules, [emqx_bridge_http_action_info]}]},

2
apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_bridge_kafka, [
{description, "EMQX Enterprise Kafka Bridge"},
{vsn, "0.1.13"},
{vsn, "0.2.0"},
{registered, [emqx_bridge_kafka_consumer_sup]},
{applications, [
kernel,

2
apps/emqx_connector/src/emqx_connector.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_connector, [
{description, "EMQX Data Integration Connectors"},
{vsn, "0.1.35"},
{vsn, "0.1.36"},
{registered, []},
{mod, {emqx_connector_app, []}},
{applications, [

25
apps/emqx_connector/src/emqx_connector_api.erl
View File

@ -348,7 +348,7 @@ schema("/connectors_probe") ->
case emqx_connector:lookup(ConnectorType, ConnectorName) of
{ok, _} ->
RawConf = emqx:get_raw_config([connectors, ConnectorType, ConnectorName], #{}),
Conf = deobfuscate(Conf1, RawConf),
Conf = emqx_utils:deobfuscate(Conf1, RawConf),
update_connector(ConnectorType, ConnectorName, Conf);
{error, not_found} ->
?CONNECTOR_NOT_FOUND(ConnectorType, ConnectorName)
@ -409,7 +409,7 @@ maybe_deobfuscate_connector_probe(
case emqx_connector:lookup(ConnectorType, ConnectorName) of
{ok, _} ->
RawConf = emqx:get_raw_config([connectors, ConnectorType, ConnectorName], #{}),
deobfuscate(Params, RawConf);
emqx_utils:deobfuscate(Params, RawConf);
_ ->
%% A connector may be probed before it's created, so not finding it here is fine
Params
@ -780,27 +780,6 @@ maybe_unwrap(RpcMulticallResult) ->
redact(Term) ->
emqx_utils:redact(Term).
deobfuscate(NewConf, OldConf) ->
maps:fold(
fun(K, V, Acc) ->
case maps:find(K, OldConf) of
error ->
Acc#{K => V};
{ok, OldV} when is_map(V), is_map(OldV) ->
Acc#{K => deobfuscate(V, OldV)};
{ok, OldV} ->
case emqx_utils:is_redacted(K, V) of
true ->
Acc#{K => OldV};
_ ->
Acc#{K => V}
end
end
end,
#{},
NewConf
).
map_to_json(M0) ->
%% When dealing with Hocon validation errors, `value' might contain non-serializable
%% values (e.g.: user_lookup_fun), so we try again without that key if serialization

2
apps/emqx_durable_storage/src/emqx_durable_storage.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_durable_storage, [
{description, "Message persistence and subscription replays for EMQX"},
% strict semver, bump manually!
{vsn, "0.1.8"},
{vsn, "0.1.9"},
{modules, []},
{registered, []},
{applications, [kernel, stdlib, rocksdb, gproc, mria, emqx_utils]},

2
apps/emqx_ft/src/emqx_ft.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_ft, [
{description, "EMQX file transfer over MQTT"},
{vsn, "0.1.10"},
{vsn, "0.1.11"},
{registered, []},
{mod, {emqx_ft_app, []}},
{applications, [

10
apps/emqx_ft/src/emqx_ft_api.erl
View File

@ -178,7 +178,9 @@ check_ft_enabled(Params, _Meta) ->
'/file_transfer'(get, _Meta) ->
{200, format_config(emqx_ft_conf:get())};
'/file_transfer'(put, #{body := ConfigIn}) ->
case emqx_ft_conf:update(ConfigIn) of
OldConf = emqx_ft_conf:get_raw(),
UpdateConf = emqx_utils:deobfuscate(ConfigIn, OldConf),
case emqx_ft_conf:update(UpdateConf) of
{ok, #{config := Config}} ->
{200, format_config(Config)};
{error, Error = #{kind := validation_error}} ->
@ -199,7 +201,11 @@ format_page(#{items := Files}) ->
format_config(Config) ->
Schema = emqx_hocon:make_schema(emqx_ft_schema:fields(file_transfer)),
hocon_tconf:make_serializable(Schema, emqx_utils_maps:binary_key_map(Config), #{}).
hocon_tconf:make_serializable(
Schema,
emqx_utils_maps:binary_key_map(Config),
#{obfuscate_sensitive_values => true}
).
format_validation_error(Error) ->
emqx_logger_jsonfmt:best_effort_json(Error).

6
apps/emqx_ft/src/emqx_ft_conf.erl
View File

@ -37,7 +37,8 @@
load/0,
unload/0,
get/0,
update/1
update/1,
get_raw/0
]).
%% callbacks for emqx_config_handler
@ -112,6 +113,9 @@ unload() ->
get() ->
emqx_config:get([file_transfer]).
get_raw() ->
emqx:get_raw_config([file_transfer], #{}).
-spec update(emqx_config:config()) -> {ok, emqx_config:update_result()} | {error, term()}.
update(Config) ->
emqx_conf:update([file_transfer], Config, #{override_to => cluster}).

96
apps/emqx_ft/test/emqx_ft_api_SUITE.erl
View File

@ -22,6 +22,8 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("stdlib/include/assert.hrl").
-define(SECRET_ACCESS_KEY, <<"fake_secret_access_key">>).
-import(emqx_dashboard_api_test_helpers, [host/0, uri/1]).
all() -> emqx_common_test_helpers:all(?MODULE).
@ -335,6 +337,7 @@ t_configure(Config) ->
<<"host">> => <<"localhost">>,
<<"port">> => 9000,
<<"bucket">> => <<"emqx">>,
<<"secret_access_key">> => ?SECRET_ACCESS_KEY,
<<"transport_options">> => #{
<<"ssl">> => #{
<<"enable">> => true,
@ -343,25 +346,7 @@ t_configure(Config) ->
}
}
},
?assertMatch(
{ok, 200, #{
<<"enable">> := true,
<<"storage">> := #{
<<"local">> := #{
<<"exporter">> := #{
<<"s3">> := #{
<<"transport_options">> := #{
<<"ssl">> := #{
<<"enable">> := true,
<<"certfile">> := <<"/", _CertFilepath/bytes>>,
<<"keyfile">> := <<"/", _KeyFilepath/bytes>>
}
}
}
}
}
}
}},
{ok, 200, GetConfigJson} =
request_json(
put,
uri(["file_transfer"]),
@ -376,7 +361,28 @@ t_configure(Config) ->
}
},
Config
)
),
?assertMatch(
#{
<<"enable">> := true,
<<"storage">> := #{
<<"local">> := #{
<<"exporter">> := #{
<<"s3">> := #{
<<"transport_options">> := #{
<<"ssl">> := #{
<<"enable">> := true,
<<"certfile">> := <<"/", _CertFilepath/bytes>>,
<<"keyfile">> := <<"/", _KeyFilepath/bytes>>
}
},
<<"secret_access_key">> := <<"******">>
}
}
}
}
},
GetConfigJson
),
?assertMatch(
{ok, 400, _},
@ -405,23 +411,47 @@ t_configure(Config) ->
request_json(
put,
uri(["file_transfer"]),
#{
<<"enable">> => true,
<<"storage">> => #{
<<"local">> => #{
<<"exporter">> => #{
<<"s3">> => emqx_utils_maps:deep_put(
[<<"transport_options">>, <<"ssl">>, <<"enable">>],
S3Exporter,
false
)
emqx_utils_maps:deep_merge(
GetConfigJson,
#{
<<"enable">> => true,
<<"storage">> => #{
<<"local">> => #{
<<"exporter">> => #{
<<"s3">> => emqx_utils_maps:deep_put(
[<<"transport_options">>, <<"ssl">>, <<"enable">>],
S3Exporter,
false
)
}
}
}
}
},
),
Config
)
),
%% put secret as ******, check the secret is unchanged
?assertMatch(
#{
<<"storage">> :=
#{
<<"local">> :=
#{
<<"enable">> := true,
<<"exporter">> :=
#{
<<"s3">> :=
#{
<<"enable">> := true,
<<"secret_access_key">> := ?SECRET_ACCESS_KEY
}
}
}
}
},
get_ft_config(Config)
),
ok.
%%--------------------------------------------------------------------
@ -500,3 +530,7 @@ reset_ft_config(Config, Enable) ->
},
{ok, _} = rpc:call(Node, emqx_ft_conf, update, [LocalConfig]),
ok.
get_ft_config(Config) ->
[Node | _] = test_nodes(Config),
rpc:call(Node, emqx_ft_conf, get_raw, []).

2
apps/emqx_gateway/src/emqx_gateway.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_gateway, [
{description, "The Gateway management application"},
{vsn, "0.1.28"},
{vsn, "0.1.29"},
{registered, []},
{mod, {emqx_gateway_app, []}},
{applications, [kernel, stdlib, emqx, emqx_auth, emqx_ctl]},

2
apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_gateway_exproto, [
{description, "ExProto Gateway"},
{vsn, "0.1.6"},
{vsn, "0.1.7"},
{registered, []},
{applications, [kernel, stdlib, grpc, emqx, emqx_gateway]},
{env, []},

2
apps/emqx_gateway_ocpp/src/emqx_gateway_ocpp.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_gateway_ocpp, [
{description, "OCPP-J 1.6 Gateway for EMQX"},
{vsn, "0.1.0"},
{vsn, "0.1.1"},
{registered, []},
{applications, [kernel, stdlib, jesse, emqx, emqx_gateway]},
{env, []},

2
apps/emqx_management/src/emqx_management.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_management, [
{description, "EMQX Management API and CLI"},
% strict semver, bump manually!
{vsn, "5.0.34"},
{vsn, "5.0.35"},
{modules, []},
{registered, [emqx_management_sup]},
{applications, [

2
apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_opentelemetry, [
{description, "OpenTelemetry for EMQX Broker"},
{vsn, "0.2.0"},
{vsn, "0.2.1"},
{registered, []},
{mod, {emqx_otel_app, []}},
{applications, [

2
apps/emqx_rule_engine/src/emqx_rule_engine.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_rule_engine, [
{description, "EMQX Rule Engine"},
% strict semver, bump manually!
{vsn, "5.0.30"},
{vsn, "5.0.31"},
{modules, []},
{registered, [emqx_rule_engine_sup, emqx_rule_engine]},
{applications, [

2
apps/emqx_utils/src/emqx_utils.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_utils, [
{description, "Miscellaneous utilities for EMQX apps"},
% strict semver, bump manually!
{vsn, "5.0.12"},
{vsn, "5.0.13"},
{modules, [
emqx_utils,
emqx_utils_api,

22
apps/emqx_utils/src/emqx_utils.erl
View File

@ -78,6 +78,7 @@
]).
-export([clamp/3, redact/1, redact/2, is_redacted/2, is_redacted/3]).
-export([deobfuscate/2]).
-export_type([
readable_error_msg/1
@ -754,6 +755,27 @@ redact_v([{str, Bin}]) when is_binary(Bin) ->
redact_v(_V) ->
?REDACT_VAL.
deobfuscate(NewConf, OldConf) ->
maps:fold(
fun(K, V, Acc) ->
case maps:find(K, OldConf) of
error ->
Acc#{K => V};
{ok, OldV} when is_map(V), is_map(OldV) ->
Acc#{K => deobfuscate(V, OldV)};
{ok, OldV} ->
case is_redacted(K, V) of
true ->
Acc#{K => OldV};
_ ->
Acc#{K => V}
end
end
end,
#{},
NewConf
).
is_redacted(K, V) ->
do_is_redacted(K, V, fun is_sensitive_key/1).