Merge pull request #12220 from zmstone/1221-prepare-bpapi-on-otp-26

1221 prepare bpapi on otp 26

.gitignore

@@ -61,7 +61,7 @@ erlang_ls.config
 .envrc
 # elixir
 mix.lock
-apps/emqx/test/emqx_static_checks_data/master.bpapi
+apps/emqx/test/emqx_static_checks_data/master.bpapi2
 # rendered configurations
 *.conf.rendered
 *.conf.rendered.*

apps/emqx/include/emqx_release.hrl

@@ -35,7 +35,7 @@
 -define(EMQX_RELEASE_CE, "5.4.0").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.4.0").
+-define(EMQX_RELEASE_EE, "5.4.0-build.1").
 
 %% The HTTP API version
 -define(EMQX_API_VERSION, "5.0").

apps/emqx/src/bpapi/README.md

@@ -145,12 +145,12 @@ free to cache it for the duration of the session.
 
 # New minor release
 
-After releasing, let's say, 5.1.0, the following actions should be performed to prepare for the next release:
+After releasing, let's say, 5.5.0, the following actions should be performed to prepare for the next release:
 
-1. Checkout 5.1.0 tag
+1. Checkout 5.5.0 tag
 1. Build the code
-1. Replace api version string `"master"` in `apps/emqx/test/emqx_static_checks_data/master.bpapi` with `"5.1"`
+1. Replace api version string `"master"` in `apps/emqx/test/emqx_static_checks_data/master.bpapi2` with `"5.5"`
-1. Rename `apps/emqx/test/emqx_static_checks_data/master.bpapi` to `apps/emqx/test/emqx_static_checks_data/5.1.bpapi`
+1. Rename `apps/emqx/test/emqx_static_checks_data/master.bpapi` to `apps/emqx/test/emqx_static_checks_data/5.5.bpapi2`
-1. Add `apps/emqx/test/emqx_static_checks_data/5.1.bpapi` to the repo
+1. Add `apps/emqx/test/emqx_static_checks_data/5.5.bpapi2` to the repo
-1. Delete the previous file (e.g. `5.0.bpapi`), unless there is plan to support rolling upgrade from 5.0 to 5.2
+1. Delete the previous file (e.g. `5.4.bpapi2`), unless there is plan to support rolling upgrade from 5.0.
 1. Merge the commit to master branch

apps/emqx/src/emqx.app.src

@@ -2,7 +2,7 @@
 {application, emqx, [
     {id, "emqx"},
     {description, "EMQX Core"},
-    {vsn, "5.1.15"},
+    {vsn, "5.1.17"},
     {modules, []},
     {registered, []},
     {applications, [

apps/emqx_auth/src/emqx_auth.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_auth, [
     {description, "EMQX Authentication and authorization"},
-    {vsn, "0.1.29"},
+    {vsn, "0.2.1"},
     {modules, []},
     {registered, [emqx_auth_sup]},
     {applications, [

apps/emqx_auth_http/src/emqx_auth_http.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_auth_http, [
     {description, "EMQX External HTTP API Authentication and Authorization"},
-    {vsn, "0.1.2"},
+    {vsn, "0.1.3"},
     {registered, []},
     {mod, {emqx_auth_http_app, []}},
     {applications, [

apps/emqx_bridge/src/emqx_bridge.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge, [
     {description, "EMQX bridges"},
-    {vsn, "0.1.31"},
+    {vsn, "0.1.32"},
     {registered, [emqx_bridge_sup]},
     {mod, {emqx_bridge_app, []}},
     {applications, [

apps/emqx_bridge/src/emqx_bridge_api.erl

@@ -495,7 +495,7 @@ schema("/bridges_probe") ->
         case emqx_bridge:lookup(BridgeType, BridgeName) of
             {ok, #{raw_config := RawConf}} ->
                 %% TODO will the maybe_upgrade step done by emqx_bridge:lookup cause any problems
-                Conf = deobfuscate(Conf1, RawConf),
+                Conf = emqx_utils:deobfuscate(Conf1, RawConf),
                 update_bridge(BridgeType, BridgeName, Conf);
             {error, not_found} ->
                 ?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
@@ -587,9 +587,9 @@ maybe_deobfuscate_bridge_probe(#{<<"type">> := BridgeType0, <<"name">> := Bridge
     BridgeType = upgrade_type(BridgeType0),
     case emqx_bridge:lookup(BridgeType, BridgeName) of
         {ok, #{raw_config := RawConf}} ->
-            %% TODO check if RawConf optained above is compatible with the commented out code below
+            %% TODO check if RawConf obtained above is compatible with the commented out code below
             %% RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}),
-            deobfuscate(Params, RawConf);
+            emqx_utils:deobfuscate(Params, RawConf);
         _ ->
             %% A bridge may be probed before it's created, so not finding it here is fine
             Params
@@ -1123,27 +1123,6 @@ supported_versions(_Call) -> [1, 2, 3, 4, 5].
 redact(Term) ->
     emqx_utils:redact(Term).
 
-deobfuscate(NewConf, OldConf) ->
-    maps:fold(
-        fun(K, V, Acc) ->
-            case maps:find(K, OldConf) of
-                error ->
-                    Acc#{K => V};
-                {ok, OldV} when is_map(V), is_map(OldV) ->
-                    Acc#{K => deobfuscate(V, OldV)};
-                {ok, OldV} ->
-                    case emqx_utils:is_redacted(K, V) of
-                        true ->
-                            Acc#{K => OldV};
-                        _ ->
-                            Acc#{K => V}
-                    end
-            end
-        end,
-        #{},
-        NewConf
-    ).
-
 map_to_json(M0) ->
     %% When dealing with Hocon validation errors, `value' might contain non-serializable
     %% values (e.g.: user_lookup_fun), so we try again without that key if serialization

apps/emqx_bridge/src/emqx_bridge_v2_api.erl

@@ -423,7 +423,7 @@ schema("/action_types") ->
         case emqx_bridge_v2:lookup(BridgeType, BridgeName) of
             {ok, _} ->
                 RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}),
-                Conf = deobfuscate(Conf1, RawConf),
+                Conf = emqx_utils:deobfuscate(Conf1, RawConf),
                 update_bridge(BridgeType, BridgeName, Conf);
             {error, not_found} ->
                 ?BRIDGE_NOT_FOUND(BridgeType, BridgeName)
@@ -554,9 +554,9 @@ schema("/action_types") ->
 maybe_deobfuscate_bridge_probe(#{<<"type">> := ActionType, <<"name">> := BridgeName} = Params) ->
     case emqx_bridge_v2:lookup(ActionType, BridgeName) of
         {ok, #{raw_config := RawConf}} ->
-            %% TODO check if RawConf optained above is compatible with the commented out code below
+            %% TODO check if RawConf obtained above is compatible with the commented out code below
             %% RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}),
-            deobfuscate(Params, RawConf);
+            emqx_utils:deobfuscate(Params, RawConf);
         _ ->
             %% A bridge may be probed before it's created, so not finding it here is fine
             Params
@@ -586,27 +586,6 @@ is_ok(ResL) ->
         ErrL -> hd(ErrL)
     end.
 
-deobfuscate(NewConf, OldConf) ->
-    maps:fold(
-        fun(K, V, Acc) ->
-            case maps:find(K, OldConf) of
-                error ->
-                    Acc#{K => V};
-                {ok, OldV} when is_map(V), is_map(OldV) ->
-                    Acc#{K => deobfuscate(V, OldV)};
-                {ok, OldV} ->
-                    case emqx_utils:is_redacted(K, V) of
-                        true ->
-                            Acc#{K => OldV};
-                        _ ->
-                            Acc#{K => V}
-                    end
-            end
-        end,
-        #{},
-        NewConf
-    ).
-
 %% bridge helpers
 lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) ->
     Nodes = mria:running_nodes(),

apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_gcp_pubsub, [
     {description, "EMQX Enterprise GCP Pub/Sub Bridge"},
-    {vsn, "0.1.11"},
+    {vsn, "0.2.1"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_greptimedb, [
     {description, "EMQX GreptimeDB Bridge"},
-    {vsn, "0.1.5"},
+    {vsn, "0.1.7"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_http/src/emqx_bridge_http.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_http, [
     {description, "EMQX HTTP Bridge and Connector Application"},
-    {vsn, "0.1.6"},
+    {vsn, "0.2.1"},
     {registered, []},
     {applications, [kernel, stdlib, emqx_connector, emqx_resource, ehttpc]},
     {env, [{emqx_action_info_modules, [emqx_bridge_http_action_info]}]},

apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge_kafka, [
     {description, "EMQX Enterprise Kafka Bridge"},
-    {vsn, "0.1.13"},
+    {vsn, "0.2.1"},
     {registered, [emqx_bridge_kafka_consumer_sup]},
     {applications, [
         kernel,

apps/emqx_connector/src/emqx_connector.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_connector, [
     {description, "EMQX Data Integration Connectors"},
-    {vsn, "0.1.35"},
+    {vsn, "0.1.36"},
     {registered, []},
     {mod, {emqx_connector_app, []}},
     {applications, [

apps/emqx_connector/src/emqx_connector_api.erl

@@ -348,7 +348,7 @@ schema("/connectors_probe") ->
         case emqx_connector:lookup(ConnectorType, ConnectorName) of
             {ok, _} ->
                 RawConf = emqx:get_raw_config([connectors, ConnectorType, ConnectorName], #{}),
-                Conf = deobfuscate(Conf1, RawConf),
+                Conf = emqx_utils:deobfuscate(Conf1, RawConf),
                 update_connector(ConnectorType, ConnectorName, Conf);
             {error, not_found} ->
                 ?CONNECTOR_NOT_FOUND(ConnectorType, ConnectorName)
@@ -409,7 +409,7 @@ maybe_deobfuscate_connector_probe(
     case emqx_connector:lookup(ConnectorType, ConnectorName) of
         {ok, _} ->
             RawConf = emqx:get_raw_config([connectors, ConnectorType, ConnectorName], #{}),
-            deobfuscate(Params, RawConf);
+            emqx_utils:deobfuscate(Params, RawConf);
         _ ->
             %% A connector may be probed before it's created, so not finding it here is fine
             Params
@@ -780,27 +780,6 @@ maybe_unwrap(RpcMulticallResult) ->
 redact(Term) ->
     emqx_utils:redact(Term).
 
-deobfuscate(NewConf, OldConf) ->
-    maps:fold(
-        fun(K, V, Acc) ->
-            case maps:find(K, OldConf) of
-                error ->
-                    Acc#{K => V};
-                {ok, OldV} when is_map(V), is_map(OldV) ->
-                    Acc#{K => deobfuscate(V, OldV)};
-                {ok, OldV} ->
-                    case emqx_utils:is_redacted(K, V) of
-                        true ->
-                            Acc#{K => OldV};
-                        _ ->
-                            Acc#{K => V}
-                    end
-            end
-        end,
-        #{},
-        NewConf
-    ).
-
 map_to_json(M0) ->
     %% When dealing with Hocon validation errors, `value' might contain non-serializable
     %% values (e.g.: user_lookup_fun), so we try again without that key if serialization

apps/emqx_durable_storage/src/emqx_durable_storage.app.src

@@ -2,7 +2,7 @@
 {application, emqx_durable_storage, [
     {description, "Message persistence and subscription replays for EMQX"},
     % strict semver, bump manually!
-    {vsn, "0.1.8"},
+    {vsn, "0.1.10"},
     {modules, []},
     {registered, []},
     {applications, [kernel, stdlib, rocksdb, gproc, mria, emqx_utils]},

apps/emqx_ft/src/emqx_ft.app.src

@@ -1,6 +1,6 @@
 {application, emqx_ft, [
     {description, "EMQX file transfer over MQTT"},
-    {vsn, "0.1.10"},
+    {vsn, "0.1.11"},
     {registered, []},
     {mod, {emqx_ft_app, []}},
     {applications, [

apps/emqx_ft/src/emqx_ft_api.erl

@@ -178,7 +178,9 @@ check_ft_enabled(Params, _Meta) ->
 '/file_transfer'(get, _Meta) ->
     {200, format_config(emqx_ft_conf:get())};
 '/file_transfer'(put, #{body := ConfigIn}) ->
-    case emqx_ft_conf:update(ConfigIn) of
+    OldConf = emqx_ft_conf:get_raw(),
+    UpdateConf = emqx_utils:deobfuscate(ConfigIn, OldConf),
+    case emqx_ft_conf:update(UpdateConf) of
         {ok, #{config := Config}} ->
             {200, format_config(Config)};
         {error, Error = #{kind := validation_error}} ->
@@ -199,7 +201,11 @@ format_page(#{items := Files}) ->
 
 format_config(Config) ->
     Schema = emqx_hocon:make_schema(emqx_ft_schema:fields(file_transfer)),
-    hocon_tconf:make_serializable(Schema, emqx_utils_maps:binary_key_map(Config), #{}).
+    hocon_tconf:make_serializable(
+        Schema,
+        emqx_utils_maps:binary_key_map(Config),
+        #{obfuscate_sensitive_values => true}
+    ).
 
 format_validation_error(Error) ->
     emqx_logger_jsonfmt:best_effort_json(Error).

apps/emqx_ft/src/emqx_ft_conf.erl

@@ -37,7 +37,8 @@
     load/0,
     unload/0,
     get/0,
-    update/1
+    update/1,
+    get_raw/0
 ]).
 
 %% callbacks for emqx_config_handler
@@ -112,6 +113,9 @@ unload() ->
 get() ->
     emqx_config:get([file_transfer]).
 
+get_raw() ->
+    emqx:get_raw_config([file_transfer], #{}).
+
 -spec update(emqx_config:config()) -> {ok, emqx_config:update_result()} | {error, term()}.
 update(Config) ->
     emqx_conf:update([file_transfer], Config, #{override_to => cluster}).

apps/emqx_ft/test/emqx_ft_api_SUITE.erl

@@ -22,6 +22,8 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("stdlib/include/assert.hrl").
 
+-define(SECRET_ACCESS_KEY, <<"fake_secret_access_key">>).
+
 -import(emqx_dashboard_api_test_helpers, [host/0, uri/1]).
 
 all() -> emqx_common_test_helpers:all(?MODULE).
@@ -335,6 +337,7 @@ t_configure(Config) ->
         <<"host">> => <<"localhost">>,
         <<"port">> => 9000,
         <<"bucket">> => <<"emqx">>,
+        <<"secret_access_key">> => ?SECRET_ACCESS_KEY,
         <<"transport_options">> => #{
             <<"ssl">> => #{
                 <<"enable">> => true,
@@ -343,25 +346,7 @@ t_configure(Config) ->
             }
         }
     },
-    ?assertMatch(
+    {ok, 200, GetConfigJson} =
-        {ok, 200, #{
-            <<"enable">> := true,
-            <<"storage">> := #{
-                <<"local">> := #{
-                    <<"exporter">> := #{
-                        <<"s3">> := #{
-                            <<"transport_options">> := #{
-                                <<"ssl">> := #{
-                                    <<"enable">> := true,
-                                    <<"certfile">> := <<"/", _CertFilepath/bytes>>,
-                                    <<"keyfile">> := <<"/", _KeyFilepath/bytes>>
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }},
         request_json(
             put,
             uri(["file_transfer"]),
@@ -376,7 +361,28 @@ t_configure(Config) ->
                 }
             },
             Config
-        )
+        ),
+    ?assertMatch(
+        #{
+            <<"enable">> := true,
+            <<"storage">> := #{
+                <<"local">> := #{
+                    <<"exporter">> := #{
+                        <<"s3">> := #{
+                            <<"transport_options">> := #{
+                                <<"ssl">> := #{
+                                    <<"enable">> := true,
+                                    <<"certfile">> := <<"/", _CertFilepath/bytes>>,
+                                    <<"keyfile">> := <<"/", _KeyFilepath/bytes>>
+                                }
+                            },
+                            <<"secret_access_key">> := <<"******">>
+                        }
+                    }
+                }
+            }
+        },
+        GetConfigJson
     ),
     ?assertMatch(
         {ok, 400, _},
@@ -405,23 +411,47 @@ t_configure(Config) ->
         request_json(
             put,
             uri(["file_transfer"]),
-            #{
+            emqx_utils_maps:deep_merge(
-                <<"enable">> => true,
+                GetConfigJson,
-                <<"storage">> => #{
+                #{
-                    <<"local">> => #{
+                    <<"enable">> => true,
-                        <<"exporter">> => #{
+                    <<"storage">> => #{
-                            <<"s3">> => emqx_utils_maps:deep_put(
+                        <<"local">> => #{
-                                [<<"transport_options">>, <<"ssl">>, <<"enable">>],
+                            <<"exporter">> => #{
-                                S3Exporter,
+                                <<"s3">> => emqx_utils_maps:deep_put(
-                                false
+                                    [<<"transport_options">>, <<"ssl">>, <<"enable">>],
-                            )
+                                    S3Exporter,
+                                    false
+                                )
+                            }
                         }
                     }
                 }
-            },
+            ),
             Config
         )
     ),
+    %% put secret as ******, check the secret is unchanged
+    ?assertMatch(
+        #{
+            <<"storage">> :=
+                #{
+                    <<"local">> :=
+                        #{
+                            <<"enable">> := true,
+                            <<"exporter">> :=
+                                #{
+                                    <<"s3">> :=
+                                        #{
+                                            <<"enable">> := true,
+                                            <<"secret_access_key">> := ?SECRET_ACCESS_KEY
+                                        }
+                                }
+                        }
+                }
+        },
+        get_ft_config(Config)
+    ),
     ok.
 
 %%--------------------------------------------------------------------
@@ -500,3 +530,7 @@ reset_ft_config(Config, Enable) ->
         },
     {ok, _} = rpc:call(Node, emqx_ft_conf, update, [LocalConfig]),
     ok.
+
+get_ft_config(Config) ->
+    [Node | _] = test_nodes(Config),
+    rpc:call(Node, emqx_ft_conf, get_raw, []).

apps/emqx_gateway/src/emqx_gateway.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_gateway, [
     {description, "The Gateway management application"},
-    {vsn, "0.1.28"},
+    {vsn, "0.1.30"},
     {registered, []},
     {mod, {emqx_gateway_app, []}},
     {applications, [kernel, stdlib, emqx, emqx_auth, emqx_ctl]},

apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_gateway_exproto, [
     {description, "ExProto Gateway"},
-    {vsn, "0.1.6"},
+    {vsn, "0.1.8"},
     {registered, []},
     {applications, [kernel, stdlib, grpc, emqx, emqx_gateway]},
     {env, []},

apps/emqx_gateway_ocpp/src/emqx_gateway_ocpp.app.src

@@ -1,6 +1,6 @@
 {application, emqx_gateway_ocpp, [
     {description, "OCPP-J 1.6 Gateway for EMQX"},
-    {vsn, "0.1.0"},
+    {vsn, "0.1.2"},
     {registered, []},
     {applications, [kernel, stdlib, jesse, emqx, emqx_gateway]},
     {env, []},

apps/emqx_management/src/emqx_management.app.src

@@ -2,7 +2,7 @@
 {application, emqx_management, [
     {description, "EMQX Management API and CLI"},
     % strict semver, bump manually!
-    {vsn, "5.0.34"},
+    {vsn, "5.0.36"},
     {modules, []},
     {registered, [emqx_management_sup]},
     {applications, [

apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src

@@ -1,6 +1,6 @@
 {application, emqx_opentelemetry, [
     {description, "OpenTelemetry for EMQX Broker"},
-    {vsn, "0.2.0"},
+    {vsn, "0.2.2"},
     {registered, []},
     {mod, {emqx_otel_app, []}},
     {applications, [

apps/emqx_rule_engine/src/emqx_rule_engine.app.src

@@ -2,7 +2,7 @@
 {application, emqx_rule_engine, [
     {description, "EMQX Rule Engine"},
     % strict semver, bump manually!
-    {vsn, "5.0.30"},
+    {vsn, "5.0.32"},
     {modules, []},
     {registered, [emqx_rule_engine_sup, emqx_rule_engine]},
     {applications, [

apps/emqx_utils/src/emqx_utils.app.src

@@ -2,7 +2,7 @@
 {application, emqx_utils, [
     {description, "Miscellaneous utilities for EMQX apps"},
     % strict semver, bump manually!
-    {vsn, "5.0.12"},
+    {vsn, "5.0.14"},
     {modules, [
         emqx_utils,
         emqx_utils_api,

apps/emqx_utils/src/emqx_utils.erl

@@ -79,6 +79,7 @@
 ]).
 
 -export([clamp/3, redact/1, redact/2, is_redacted/2, is_redacted/3]).
+-export([deobfuscate/2]).
 
 -export_type([
     readable_error_msg/1
@@ -755,6 +756,27 @@ redact_v([{str, Bin}]) when is_binary(Bin) ->
 redact_v(_V) ->
     ?REDACT_VAL.
 
+deobfuscate(NewConf, OldConf) ->
+    maps:fold(
+        fun(K, V, Acc) ->
+            case maps:find(K, OldConf) of
+                error ->
+                    Acc#{K => V};
+                {ok, OldV} when is_map(V), is_map(OldV) ->
+                    Acc#{K => deobfuscate(V, OldV)};
+                {ok, OldV} ->
+                    case is_redacted(K, V) of
+                        true ->
+                            Acc#{K => OldV};
+                        _ ->
+                            Acc#{K => V}
+                    end
+            end
+        end,
+        #{},
+        NewConf
+    ).
+
 is_redacted(K, V) ->
     do_is_redacted(K, V, fun is_sensitive_key/1).
 

changes/e5.4.0-build.1.en.md

@@ -118,3 +118,5 @@
 - [#12176](https://github.com/emqx/emqx/pull/12176) Always acknowledge `DISCONNECT` packet to MQTT-SN client regardless of whether the connection has been successfully established before.
 
 - [#12180](https://github.com/emqx/emqx/pull/12180) Fix an issue where DTLS enabled MQTT-SN gateways could not be started, caused by incompatibility of default listener configuration with the DTLS implementation.
+
+- [#12219](https://github.com/emqx/emqx/pull/12219) Fix file transfer S3 config secret deobfuscation issue while performing config updates from dashboard.

deploy/charts/emqx-enterprise/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.4.0
+version: 5.4.0-build.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.4.0
+appVersion: 5.4.0-build.1