Merge pull request #12219 from zhongwencool/ft-deobfuscate-keys

fix: FT deobfuscate secret key

apps/emqx/src/emqx.app.src

@@ -2,7 +2,7 @@
 {application, emqx, [
     {id, "emqx"},
     {description, "EMQX Core"},
-    {vsn, "5.1.15"},
+    {vsn, "5.1.16"},
     {modules, []},
     {registered, []},
     {applications, [

apps/emqx_auth/src/emqx_auth.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_auth, [
     {description, "EMQX Authentication and authorization"},
-    {vsn, "0.1.29"},
+    {vsn, "0.2.0"},
     {modules, []},
     {registered, [emqx_auth_sup]},
     {applications, [

apps/emqx_auth_http/src/emqx_auth_http.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_auth_http, [
     {description, "EMQX External HTTP API Authentication and Authorization"},
-    {vsn, "0.1.2"},
+    {vsn, "0.1.3"},
     {registered, []},
     {mod, {emqx_auth_http_app, []}},
     {applications, [

apps/emqx_bridge/src/emqx_bridge.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge, [
     {description, "EMQX bridges"},
-    {vsn, "0.1.31"},
+    {vsn, "0.1.32"},
     {registered, [emqx_bridge_sup]},
     {mod, {emqx_bridge_app, []}},
     {applications, [

apps/emqx_bridge/src/emqx_bridge_api.erl

@@ -495,7 +495,7 @@ schema("/bridges_probe") ->
         case emqx_bridge:lookup(BridgeType, BridgeName) of
             {ok, #{raw_config := RawConf}} ->
                 %% TODO will the maybe_upgrade step done by emqx_bridge:lookup cause any problems
-                Conf = deobfuscate(Conf1, RawConf),
+                Conf = emqx_utils:deobfuscate(Conf1, RawConf),
                 update_bridge(BridgeType, BridgeName, Conf);
             {error, not_found} ->
                 ?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
@@ -587,9 +587,9 @@ maybe_deobfuscate_bridge_probe(#{<<"type">> := BridgeType0, <<"name">> := Bridge
     BridgeType = upgrade_type(BridgeType0),
     case emqx_bridge:lookup(BridgeType, BridgeName) of
         {ok, #{raw_config := RawConf}} ->
-            %% TODO check if RawConf optained above is compatible with the commented out code below
+            %% TODO check if RawConf obtained above is compatible with the commented out code below
             %% RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}),
-            deobfuscate(Params, RawConf);
+            emqx_utils:deobfuscate(Params, RawConf);
         _ ->
             %% A bridge may be probed before it's created, so not finding it here is fine
             Params
@@ -1123,27 +1123,6 @@ supported_versions(_Call) -> [1, 2, 3, 4, 5].
 redact(Term) ->
     emqx_utils:redact(Term).
 
-deobfuscate(NewConf, OldConf) ->
-    maps:fold(
-        fun(K, V, Acc) ->
-            case maps:find(K, OldConf) of
-                error ->
-                    Acc#{K => V};
-                {ok, OldV} when is_map(V), is_map(OldV) ->
-                    Acc#{K => deobfuscate(V, OldV)};
-                {ok, OldV} ->
-                    case emqx_utils:is_redacted(K, V) of
-                        true ->
-                            Acc#{K => OldV};
-                        _ ->
-                            Acc#{K => V}
-                    end
-            end
-        end,
-        #{},
-        NewConf
-    ).
-
 map_to_json(M0) ->
     %% When dealing with Hocon validation errors, `value' might contain non-serializable
     %% values (e.g.: user_lookup_fun), so we try again without that key if serialization

apps/emqx_bridge/src/emqx_bridge_v2_api.erl

@@ -423,7 +423,7 @@ schema("/action_types") ->
         case emqx_bridge_v2:lookup(BridgeType, BridgeName) of
             {ok, _} ->
                 RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}),
-                Conf = deobfuscate(Conf1, RawConf),
+                Conf = emqx_utils:deobfuscate(Conf1, RawConf),
                 update_bridge(BridgeType, BridgeName, Conf);
             {error, not_found} ->
                 ?BRIDGE_NOT_FOUND(BridgeType, BridgeName)
@@ -554,9 +554,9 @@ schema("/action_types") ->
 maybe_deobfuscate_bridge_probe(#{<<"type">> := ActionType, <<"name">> := BridgeName} = Params) ->
     case emqx_bridge_v2:lookup(ActionType, BridgeName) of
         {ok, #{raw_config := RawConf}} ->
-            %% TODO check if RawConf optained above is compatible with the commented out code below
+            %% TODO check if RawConf obtained above is compatible with the commented out code below
             %% RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}),
-            deobfuscate(Params, RawConf);
+            emqx_utils:deobfuscate(Params, RawConf);
         _ ->
             %% A bridge may be probed before it's created, so not finding it here is fine
             Params
@@ -586,27 +586,6 @@ is_ok(ResL) ->
         ErrL -> hd(ErrL)
     end.
 
-deobfuscate(NewConf, OldConf) ->
-    maps:fold(
-        fun(K, V, Acc) ->
-            case maps:find(K, OldConf) of
-                error ->
-                    Acc#{K => V};
-                {ok, OldV} when is_map(V), is_map(OldV) ->
-                    Acc#{K => deobfuscate(V, OldV)};
-                {ok, OldV} ->
-                    case emqx_utils:is_redacted(K, V) of
-                        true ->
-                            Acc#{K => OldV};
-                        _ ->
-                            Acc#{K => V}
-                    end
-            end
-        end,
-        #{},
-        NewConf
-    ).
-
 %% bridge helpers
 lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) ->
     Nodes = mria:running_nodes(),

apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_gcp_pubsub, [
     {description, "EMQX Enterprise GCP Pub/Sub Bridge"},
-    {vsn, "0.1.11"},
+    {vsn, "0.2.0"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_greptimedb, [
     {description, "EMQX GreptimeDB Bridge"},
-    {vsn, "0.1.5"},
+    {vsn, "0.1.6"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_http/src/emqx_bridge_http.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_http, [
     {description, "EMQX HTTP Bridge and Connector Application"},
-    {vsn, "0.1.6"},
+    {vsn, "0.2.0"},
     {registered, []},
     {applications, [kernel, stdlib, emqx_connector, emqx_resource, ehttpc]},
     {env, [{emqx_action_info_modules, [emqx_bridge_http_action_info]}]},

apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge_kafka, [
     {description, "EMQX Enterprise Kafka Bridge"},
-    {vsn, "0.1.13"},
+    {vsn, "0.2.0"},
     {registered, [emqx_bridge_kafka_consumer_sup]},
     {applications, [
         kernel,

apps/emqx_connector/src/emqx_connector.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_connector, [
     {description, "EMQX Data Integration Connectors"},
-    {vsn, "0.1.35"},
+    {vsn, "0.1.36"},
     {registered, []},
     {mod, {emqx_connector_app, []}},
     {applications, [

apps/emqx_connector/src/emqx_connector_api.erl

@@ -348,7 +348,7 @@ schema("/connectors_probe") ->
         case emqx_connector:lookup(ConnectorType, ConnectorName) of
             {ok, _} ->
                 RawConf = emqx:get_raw_config([connectors, ConnectorType, ConnectorName], #{}),
-                Conf = deobfuscate(Conf1, RawConf),
+                Conf = emqx_utils:deobfuscate(Conf1, RawConf),
                 update_connector(ConnectorType, ConnectorName, Conf);
             {error, not_found} ->
                 ?CONNECTOR_NOT_FOUND(ConnectorType, ConnectorName)
@@ -409,7 +409,7 @@ maybe_deobfuscate_connector_probe(
     case emqx_connector:lookup(ConnectorType, ConnectorName) of
         {ok, _} ->
             RawConf = emqx:get_raw_config([connectors, ConnectorType, ConnectorName], #{}),
-            deobfuscate(Params, RawConf);
+            emqx_utils:deobfuscate(Params, RawConf);
         _ ->
             %% A connector may be probed before it's created, so not finding it here is fine
             Params
@@ -780,27 +780,6 @@ maybe_unwrap(RpcMulticallResult) ->
 redact(Term) ->
     emqx_utils:redact(Term).
 
-deobfuscate(NewConf, OldConf) ->
-    maps:fold(
-        fun(K, V, Acc) ->
-            case maps:find(K, OldConf) of
-                error ->
-                    Acc#{K => V};
-                {ok, OldV} when is_map(V), is_map(OldV) ->
-                    Acc#{K => deobfuscate(V, OldV)};
-                {ok, OldV} ->
-                    case emqx_utils:is_redacted(K, V) of
-                        true ->
-                            Acc#{K => OldV};
-                        _ ->
-                            Acc#{K => V}
-                    end
-            end
-        end,
-        #{},
-        NewConf
-    ).
-
 map_to_json(M0) ->
     %% When dealing with Hocon validation errors, `value' might contain non-serializable
     %% values (e.g.: user_lookup_fun), so we try again without that key if serialization

apps/emqx_durable_storage/src/emqx_durable_storage.app.src

@@ -2,7 +2,7 @@
 {application, emqx_durable_storage, [
     {description, "Message persistence and subscription replays for EMQX"},
     % strict semver, bump manually!
-    {vsn, "0.1.8"},
+    {vsn, "0.1.9"},
     {modules, []},
     {registered, []},
     {applications, [kernel, stdlib, rocksdb, gproc, mria, emqx_utils]},

apps/emqx_ft/src/emqx_ft.app.src

@@ -1,6 +1,6 @@
 {application, emqx_ft, [
     {description, "EMQX file transfer over MQTT"},
-    {vsn, "0.1.10"},
+    {vsn, "0.1.11"},
     {registered, []},
     {mod, {emqx_ft_app, []}},
     {applications, [

apps/emqx_ft/src/emqx_ft_api.erl

@@ -178,7 +178,9 @@ check_ft_enabled(Params, _Meta) ->
 '/file_transfer'(get, _Meta) ->
     {200, format_config(emqx_ft_conf:get())};
 '/file_transfer'(put, #{body := ConfigIn}) ->
-    case emqx_ft_conf:update(ConfigIn) of
+    OldConf = emqx_ft_conf:get_raw(),
+    UpdateConf = emqx_utils:deobfuscate(ConfigIn, OldConf),
+    case emqx_ft_conf:update(UpdateConf) of
         {ok, #{config := Config}} ->
             {200, format_config(Config)};
         {error, Error = #{kind := validation_error}} ->
@@ -199,7 +201,11 @@ format_page(#{items := Files}) ->
 
 format_config(Config) ->
     Schema = emqx_hocon:make_schema(emqx_ft_schema:fields(file_transfer)),
-    hocon_tconf:make_serializable(Schema, emqx_utils_maps:binary_key_map(Config), #{}).
+    hocon_tconf:make_serializable(
+        Schema,
+        emqx_utils_maps:binary_key_map(Config),
+        #{obfuscate_sensitive_values => true}
+    ).
 
 format_validation_error(Error) ->
     emqx_logger_jsonfmt:best_effort_json(Error).

apps/emqx_ft/src/emqx_ft_conf.erl

@@ -37,7 +37,8 @@
     load/0,
     unload/0,
     get/0,
-    update/1
+    update/1,
+    get_raw/0
 ]).
 
 %% callbacks for emqx_config_handler
@@ -112,6 +113,9 @@ unload() ->
 get() ->
     emqx_config:get([file_transfer]).
 
+get_raw() ->
+    emqx:get_raw_config([file_transfer], #{}).
+
 -spec update(emqx_config:config()) -> {ok, emqx_config:update_result()} | {error, term()}.
 update(Config) ->
     emqx_conf:update([file_transfer], Config, #{override_to => cluster}).

apps/emqx_ft/test/emqx_ft_api_SUITE.erl

@@ -22,6 +22,8 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("stdlib/include/assert.hrl").
 
+-define(SECRET_ACCESS_KEY, <<"fake_secret_access_key">>).
+
 -import(emqx_dashboard_api_test_helpers, [host/0, uri/1]).
 
 all() -> emqx_common_test_helpers:all(?MODULE).
@@ -335,6 +337,7 @@ t_configure(Config) ->
         <<"host">> => <<"localhost">>,
         <<"port">> => 9000,
         <<"bucket">> => <<"emqx">>,
+        <<"secret_access_key">> => ?SECRET_ACCESS_KEY,
         <<"transport_options">> => #{
             <<"ssl">> => #{
                 <<"enable">> => true,
@@ -343,25 +346,7 @@ t_configure(Config) ->
             }
         }
     },
-    ?assertMatch(
+    {ok, 200, GetConfigJson} =
-        {ok, 200, #{
-            <<"enable">> := true,
-            <<"storage">> := #{
-                <<"local">> := #{
-                    <<"exporter">> := #{
-                        <<"s3">> := #{
-                            <<"transport_options">> := #{
-                                <<"ssl">> := #{
-                                    <<"enable">> := true,
-                                    <<"certfile">> := <<"/", _CertFilepath/bytes>>,
-                                    <<"keyfile">> := <<"/", _KeyFilepath/bytes>>
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }},
         request_json(
             put,
             uri(["file_transfer"]),
@@ -376,7 +361,28 @@ t_configure(Config) ->
                 }
             },
             Config
-        )
+        ),
+    ?assertMatch(
+        #{
+            <<"enable">> := true,
+            <<"storage">> := #{
+                <<"local">> := #{
+                    <<"exporter">> := #{
+                        <<"s3">> := #{
+                            <<"transport_options">> := #{
+                                <<"ssl">> := #{
+                                    <<"enable">> := true,
+                                    <<"certfile">> := <<"/", _CertFilepath/bytes>>,
+                                    <<"keyfile">> := <<"/", _KeyFilepath/bytes>>
+                                }
+                            },
+                            <<"secret_access_key">> := <<"******">>
+                        }
+                    }
+                }
+            }
+        },
+        GetConfigJson
     ),
     ?assertMatch(
         {ok, 400, _},
@@ -405,6 +411,8 @@ t_configure(Config) ->
         request_json(
             put,
             uri(["file_transfer"]),
+            emqx_utils_maps:deep_merge(
+                GetConfigJson,
                 #{
                     <<"enable">> => true,
                     <<"storage">> => #{
@@ -418,10 +426,32 @@ t_configure(Config) ->
                             }
                         }
                     }
-            },
+                }
+            ),
             Config
         )
     ),
+    %% put secret as ******, check the secret is unchanged
+    ?assertMatch(
+        #{
+            <<"storage">> :=
+                #{
+                    <<"local">> :=
+                        #{
+                            <<"enable">> := true,
+                            <<"exporter">> :=
+                                #{
+                                    <<"s3">> :=
+                                        #{
+                                            <<"enable">> := true,
+                                            <<"secret_access_key">> := ?SECRET_ACCESS_KEY
+                                        }
+                                }
+                        }
+                }
+        },
+        get_ft_config(Config)
+    ),
     ok.
 
 %%--------------------------------------------------------------------
@@ -500,3 +530,7 @@ reset_ft_config(Config, Enable) ->
         },
     {ok, _} = rpc:call(Node, emqx_ft_conf, update, [LocalConfig]),
     ok.
+
+get_ft_config(Config) ->
+    [Node | _] = test_nodes(Config),
+    rpc:call(Node, emqx_ft_conf, get_raw, []).

apps/emqx_gateway/src/emqx_gateway.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_gateway, [
     {description, "The Gateway management application"},
-    {vsn, "0.1.28"},
+    {vsn, "0.1.29"},
     {registered, []},
     {mod, {emqx_gateway_app, []}},
     {applications, [kernel, stdlib, emqx, emqx_auth, emqx_ctl]},

apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_gateway_exproto, [
     {description, "ExProto Gateway"},
-    {vsn, "0.1.6"},
+    {vsn, "0.1.7"},
     {registered, []},
     {applications, [kernel, stdlib, grpc, emqx, emqx_gateway]},
     {env, []},

apps/emqx_gateway_ocpp/src/emqx_gateway_ocpp.app.src

@@ -1,6 +1,6 @@
 {application, emqx_gateway_ocpp, [
     {description, "OCPP-J 1.6 Gateway for EMQX"},
-    {vsn, "0.1.0"},
+    {vsn, "0.1.1"},
     {registered, []},
     {applications, [kernel, stdlib, jesse, emqx, emqx_gateway]},
     {env, []},

apps/emqx_management/src/emqx_management.app.src

@@ -2,7 +2,7 @@
 {application, emqx_management, [
     {description, "EMQX Management API and CLI"},
     % strict semver, bump manually!
-    {vsn, "5.0.34"},
+    {vsn, "5.0.35"},
     {modules, []},
     {registered, [emqx_management_sup]},
     {applications, [

apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src

@@ -1,6 +1,6 @@
 {application, emqx_opentelemetry, [
     {description, "OpenTelemetry for EMQX Broker"},
-    {vsn, "0.2.0"},
+    {vsn, "0.2.1"},
     {registered, []},
     {mod, {emqx_otel_app, []}},
     {applications, [

apps/emqx_rule_engine/src/emqx_rule_engine.app.src

@@ -2,7 +2,7 @@
 {application, emqx_rule_engine, [
     {description, "EMQX Rule Engine"},
     % strict semver, bump manually!
-    {vsn, "5.0.30"},
+    {vsn, "5.0.31"},
     {modules, []},
     {registered, [emqx_rule_engine_sup, emqx_rule_engine]},
     {applications, [

apps/emqx_utils/src/emqx_utils.app.src

@@ -2,7 +2,7 @@
 {application, emqx_utils, [
     {description, "Miscellaneous utilities for EMQX apps"},
     % strict semver, bump manually!
-    {vsn, "5.0.12"},
+    {vsn, "5.0.13"},
     {modules, [
         emqx_utils,
         emqx_utils_api,

apps/emqx_utils/src/emqx_utils.erl

@@ -78,6 +78,7 @@
 ]).
 
 -export([clamp/3, redact/1, redact/2, is_redacted/2, is_redacted/3]).
+-export([deobfuscate/2]).
 
 -export_type([
     readable_error_msg/1
@@ -754,6 +755,27 @@ redact_v([{str, Bin}]) when is_binary(Bin) ->
 redact_v(_V) ->
     ?REDACT_VAL.
 
+deobfuscate(NewConf, OldConf) ->
+    maps:fold(
+        fun(K, V, Acc) ->
+            case maps:find(K, OldConf) of
+                error ->
+                    Acc#{K => V};
+                {ok, OldV} when is_map(V), is_map(OldV) ->
+                    Acc#{K => deobfuscate(V, OldV)};
+                {ok, OldV} ->
+                    case is_redacted(K, V) of
+                        true ->
+                            Acc#{K => OldV};
+                        _ ->
+                            Acc#{K => V}
+                    end
+            end
+        end,
+        #{},
+        NewConf
+    ).
+
 is_redacted(K, V) ->
     do_is_redacted(K, V, fun is_sensitive_key/1).