yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/release-54' into 1221-prepare-bpapi-on-otp-26

This commit is contained in:
Zaiming (Stone) Shi 2023-12-22 15:19:25 +01:00
parent 322b7bb7d2 8530a18285
commit 891fd972bd
13 changed files with 116 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/emqx/include/emqx_release.hrl
View File

@ -35,7 +35,7 @@
-define(EMQX_RELEASE_CE, "5.4.0").
%% Enterprise edition
-define(EMQX_RELEASE_EE, "5.4.0").
-define(EMQX_RELEASE_EE, "5.4.0-build.1").
%% The HTTP API version
-define(EMQX_API_VERSION, "5.0").

2
apps/emqx_bridge/src/emqx_bridge.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_bridge, [
{description, "EMQX bridges"},
{vsn, "0.1.31"},
{vsn, "0.1.32"},
{registered, [emqx_bridge_sup]},
{mod, {emqx_bridge_app, []}},
{applications, [

27
apps/emqx_bridge/src/emqx_bridge_api.erl
View File

@ -495,7 +495,7 @@ schema("/bridges_probe") ->
case emqx_bridge:lookup(BridgeType, BridgeName) of
{ok, #{raw_config := RawConf}} ->
%% TODO will the maybe_upgrade step done by emqx_bridge:lookup cause any problems
Conf = deobfuscate(Conf1, RawConf),
Conf = emqx_utils:deobfuscate(Conf1, RawConf),
update_bridge(BridgeType, BridgeName, Conf);
{error, not_found} ->
?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
@ -587,9 +587,9 @@ maybe_deobfuscate_bridge_probe(#{<<"type">> := BridgeType0, <<"name">> := Bridge
BridgeType = upgrade_type(BridgeType0),
case emqx_bridge:lookup(BridgeType, BridgeName) of
{ok, #{raw_config := RawConf}} ->
%% TODO check if RawConf optained above is compatible with the commented out code below
%% TODO check if RawConf obtained above is compatible with the commented out code below
%% RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}),
deobfuscate(Params, RawConf);
emqx_utils:deobfuscate(Params, RawConf);
_ ->
%% A bridge may be probed before it's created, so not finding it here is fine
Params
@ -1123,27 +1123,6 @@ supported_versions(_Call) -> [1, 2, 3, 4, 5].
redact(Term) ->
emqx_utils:redact(Term).
deobfuscate(NewConf, OldConf) ->
maps:fold(
fun(K, V, Acc) ->
case maps:find(K, OldConf) of
error ->
Acc#{K => V};
{ok, OldV} when is_map(V), is_map(OldV) ->
Acc#{K => deobfuscate(V, OldV)};
{ok, OldV} ->
case emqx_utils:is_redacted(K, V) of
true ->
Acc#{K => OldV};
_ ->
Acc#{K => V}
end
end
end,
#{},
NewConf
).
map_to_json(M0) ->
%% When dealing with Hocon validation errors, `value' might contain non-serializable
%% values (e.g.: user_lookup_fun), so we try again without that key if serialization

27
apps/emqx_bridge/src/emqx_bridge_v2_api.erl
View File

@ -423,7 +423,7 @@ schema("/action_types") ->
case emqx_bridge_v2:lookup(BridgeType, BridgeName) of
{ok, _} ->
RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}),
Conf = deobfuscate(Conf1, RawConf),
Conf = emqx_utils:deobfuscate(Conf1, RawConf),
update_bridge(BridgeType, BridgeName, Conf);
{error, not_found} ->
?BRIDGE_NOT_FOUND(BridgeType, BridgeName)
@ -554,9 +554,9 @@ schema("/action_types") ->
maybe_deobfuscate_bridge_probe(#{<<"type">> := ActionType, <<"name">> := BridgeName} = Params) ->
case emqx_bridge_v2:lookup(ActionType, BridgeName) of
{ok, #{raw_config := RawConf}} ->
%% TODO check if RawConf optained above is compatible with the commented out code below
%% TODO check if RawConf obtained above is compatible with the commented out code below
%% RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}),
deobfuscate(Params, RawConf);
emqx_utils:deobfuscate(Params, RawConf);
_ ->
%% A bridge may be probed before it's created, so not finding it here is fine
Params
@ -586,27 +586,6 @@ is_ok(ResL) ->
ErrL -> hd(ErrL)
end.
deobfuscate(NewConf, OldConf) ->
maps:fold(
fun(K, V, Acc) ->
case maps:find(K, OldConf) of
error ->
Acc#{K => V};
{ok, OldV} when is_map(V), is_map(OldV) ->
Acc#{K => deobfuscate(V, OldV)};
{ok, OldV} ->
case emqx_utils:is_redacted(K, V) of
true ->
Acc#{K => OldV};
_ ->
Acc#{K => V}
end
end
end,
#{},
NewConf
).
%% bridge helpers
lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) ->
Nodes = mria:running_nodes(),

2
apps/emqx_connector/src/emqx_connector.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_connector, [
{description, "EMQX Data Integration Connectors"},
{vsn, "0.1.35"},
{vsn, "0.1.36"},
{registered, []},
{mod, {emqx_connector_app, []}},
{applications, [

25
apps/emqx_connector/src/emqx_connector_api.erl
View File

@ -348,7 +348,7 @@ schema("/connectors_probe") ->
case emqx_connector:lookup(ConnectorType, ConnectorName) of
{ok, _} ->
RawConf = emqx:get_raw_config([connectors, ConnectorType, ConnectorName], #{}),
Conf = deobfuscate(Conf1, RawConf),
Conf = emqx_utils:deobfuscate(Conf1, RawConf),
update_connector(ConnectorType, ConnectorName, Conf);
{error, not_found} ->
?CONNECTOR_NOT_FOUND(ConnectorType, ConnectorName)
@ -409,7 +409,7 @@ maybe_deobfuscate_connector_probe(
case emqx_connector:lookup(ConnectorType, ConnectorName) of
{ok, _} ->
RawConf = emqx:get_raw_config([connectors, ConnectorType, ConnectorName], #{}),
deobfuscate(Params, RawConf);
emqx_utils:deobfuscate(Params, RawConf);
_ ->
%% A connector may be probed before it's created, so not finding it here is fine
Params
@ -780,27 +780,6 @@ maybe_unwrap(RpcMulticallResult) ->
redact(Term) ->
emqx_utils:redact(Term).
deobfuscate(NewConf, OldConf) ->
maps:fold(
fun(K, V, Acc) ->
case maps:find(K, OldConf) of
error ->
Acc#{K => V};
{ok, OldV} when is_map(V), is_map(OldV) ->
Acc#{K => deobfuscate(V, OldV)};
{ok, OldV} ->
case emqx_utils:is_redacted(K, V) of
true ->
Acc#{K => OldV};
_ ->
Acc#{K => V}
end
end
end,
#{},
NewConf
).
map_to_json(M0) ->
%% When dealing with Hocon validation errors, `value' might contain non-serializable
%% values (e.g.: user_lookup_fun), so we try again without that key if serialization

2
apps/emqx_ft/src/emqx_ft.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_ft, [
{description, "EMQX file transfer over MQTT"},
{vsn, "0.1.10"},
{vsn, "0.1.11"},
{registered, []},
{mod, {emqx_ft_app, []}},
{applications, [

10
apps/emqx_ft/src/emqx_ft_api.erl
View File

@ -178,7 +178,9 @@ check_ft_enabled(Params, _Meta) ->
'/file_transfer'(get, _Meta) ->
{200, format_config(emqx_ft_conf:get())};
'/file_transfer'(put, #{body := ConfigIn}) ->
case emqx_ft_conf:update(ConfigIn) of
OldConf = emqx_ft_conf:get_raw(),
UpdateConf = emqx_utils:deobfuscate(ConfigIn, OldConf),
case emqx_ft_conf:update(UpdateConf) of
{ok, #{config := Config}} ->
{200, format_config(Config)};
{error, Error = #{kind := validation_error}} ->
@ -199,7 +201,11 @@ format_page(#{items := Files}) ->
format_config(Config) ->
Schema = emqx_hocon:make_schema(emqx_ft_schema:fields(file_transfer)),
hocon_tconf:make_serializable(Schema, emqx_utils_maps:binary_key_map(Config), #{}).
hocon_tconf:make_serializable(
Schema,
emqx_utils_maps:binary_key_map(Config),
#{obfuscate_sensitive_values => true}
).
format_validation_error(Error) ->
emqx_logger_jsonfmt:best_effort_json(Error).

6
apps/emqx_ft/src/emqx_ft_conf.erl
View File

@ -37,7 +37,8 @@
load/0,
unload/0,
get/0,
update/1
update/1,
get_raw/0
]).
%% callbacks for emqx_config_handler
@ -112,6 +113,9 @@ unload() ->
get() ->
emqx_config:get([file_transfer]).
get_raw() ->
emqx:get_raw_config([file_transfer], #{}).
-spec update(emqx_config:config()) -> {ok, emqx_config:update_result()} | {error, term()}.
update(Config) ->
emqx_conf:update([file_transfer], Config, #{override_to => cluster}).

96
apps/emqx_ft/test/emqx_ft_api_SUITE.erl
View File

@ -22,6 +22,8 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("stdlib/include/assert.hrl").
-define(SECRET_ACCESS_KEY, <<"fake_secret_access_key">>).
-import(emqx_dashboard_api_test_helpers, [host/0, uri/1]).
all() -> emqx_common_test_helpers:all(?MODULE).
@ -335,6 +337,7 @@ t_configure(Config) ->
<<"host">> => <<"localhost">>,
<<"port">> => 9000,
<<"bucket">> => <<"emqx">>,
<<"secret_access_key">> => ?SECRET_ACCESS_KEY,
<<"transport_options">> => #{
<<"ssl">> => #{
<<"enable">> => true,
@ -343,25 +346,7 @@ t_configure(Config) ->
}
}
},
?assertMatch(
{ok, 200, #{
<<"enable">> := true,
<<"storage">> := #{
<<"local">> := #{
<<"exporter">> := #{
<<"s3">> := #{
<<"transport_options">> := #{
<<"ssl">> := #{
<<"enable">> := true,
<<"certfile">> := <<"/", _CertFilepath/bytes>>,
<<"keyfile">> := <<"/", _KeyFilepath/bytes>>
}
}
}
}
}
}
}},
{ok, 200, GetConfigJson} =
request_json(
put,
uri(["file_transfer"]),
@ -376,7 +361,28 @@ t_configure(Config) ->
}
},
Config
)
),
?assertMatch(
#{
<<"enable">> := true,
<<"storage">> := #{
<<"local">> := #{
<<"exporter">> := #{
<<"s3">> := #{
<<"transport_options">> := #{
<<"ssl">> := #{
<<"enable">> := true,
<<"certfile">> := <<"/", _CertFilepath/bytes>>,
<<"keyfile">> := <<"/", _KeyFilepath/bytes>>
}
},
<<"secret_access_key">> := <<"******">>
}
}
}
}
},
GetConfigJson
),
?assertMatch(
{ok, 400, _},
@ -405,23 +411,47 @@ t_configure(Config) ->
request_json(
put,
uri(["file_transfer"]),
#{
<<"enable">> => true,
<<"storage">> => #{
<<"local">> => #{
<<"exporter">> => #{
<<"s3">> => emqx_utils_maps:deep_put(
[<<"transport_options">>, <<"ssl">>, <<"enable">>],
S3Exporter,
false
)
emqx_utils_maps:deep_merge(
GetConfigJson,
#{
<<"enable">> => true,
<<"storage">> => #{
<<"local">> => #{
<<"exporter">> => #{
<<"s3">> => emqx_utils_maps:deep_put(
[<<"transport_options">>, <<"ssl">>, <<"enable">>],
S3Exporter,
false
)
}
}
}
}
},
),
Config
)
),
%% put secret as ******, check the secret is unchanged
?assertMatch(
#{
<<"storage">> :=
#{
<<"local">> :=
#{
<<"enable">> := true,
<<"exporter">> :=
#{
<<"s3">> :=
#{
<<"enable">> := true,
<<"secret_access_key">> := ?SECRET_ACCESS_KEY
}
}
}
}
},
get_ft_config(Config)
),
ok.
%%--------------------------------------------------------------------
@ -500,3 +530,7 @@ reset_ft_config(Config, Enable) ->
},
{ok, _} = rpc:call(Node, emqx_ft_conf, update, [LocalConfig]),
ok.
get_ft_config(Config) ->
[Node | _] = test_nodes(Config),
rpc:call(Node, emqx_ft_conf, get_raw, []).

22
apps/emqx_utils/src/emqx_utils.erl
View File

@ -79,6 +79,7 @@
]).
-export([clamp/3, redact/1, redact/2, is_redacted/2, is_redacted/3]).
-export([deobfuscate/2]).
-export_type([
readable_error_msg/1
@ -755,6 +756,27 @@ redact_v([{str, Bin}]) when is_binary(Bin) ->
redact_v(_V) ->
?REDACT_VAL.
deobfuscate(NewConf, OldConf) ->
maps:fold(
fun(K, V, Acc) ->
case maps:find(K, OldConf) of
error ->
Acc#{K => V};
{ok, OldV} when is_map(V), is_map(OldV) ->
Acc#{K => deobfuscate(V, OldV)};
{ok, OldV} ->
case is_redacted(K, V) of
true ->
Acc#{K => OldV};
_ ->
Acc#{K => V}
end
end
end,
#{},
NewConf
).
is_redacted(K, V) ->
do_is_redacted(K, V, fun is_sensitive_key/1).

2
changes/e5.4.0.en.md → changes/e5.4.0-build.1.en.md
View File

@ -118,3 +118,5 @@
- [#12176](https://github.com/emqx/emqx/pull/12176) Always acknowledge `DISCONNECT` packet to MQTT-SN client regardless of whether the connection has been successfully established before.
- [#12180](https://github.com/emqx/emqx/pull/12180) Fix an issue where DTLS enabled MQTT-SN gateways could not be started, caused by incompatibility of default listener configuration with the DTLS implementation.
- [#12219](https://github.com/emqx/emqx/pull/12219) Fix file transfer S3 config secret deobfuscation issue while performing config updates from dashboard.

4
deploy/charts/emqx-enterprise/Chart.yaml
View File

@ -14,8 +14,8 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
version: 5.4.0
version: 5.4.0-build.1
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application.
appVersion: 5.4.0
appVersion: 5.4.0-build.1