yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into tag-v5026

This commit is contained in:
Thales Macedo Garitezi 2023-05-29 09:15:48 -03:00
parent 6d264bb79c 6551eb21f2
commit 639feefab6
14 changed files with 128 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/emqx_authn/src/emqx_authn.erl
View File

@ -106,7 +106,7 @@ get_enabled_authns() ->
AuthnTypes = lists:usort([
Type
|| #{authenticators := As} <- Chains,
#{id := Type} <- As
#{id := Type, enable := true} <- As
]),
OverriddenListeners =
lists:foldl(

2
apps/emqx_authz/src/emqx_authz.erl
View File

@ -417,7 +417,7 @@ do_authorize(
end.
get_enabled_authzs() ->
lists:usort([Type || #{type := Type} <- lookup()]).
lists:usort([Type || #{type := Type, enable := true} <- lookup()]).
%%--------------------------------------------------------------------
%% Internal function

2
apps/emqx_authz/test/emqx_authz_SUITE.erl
View File

@ -366,7 +366,7 @@ t_get_enabled_authzs_none_enabled(_Config) ->
?assertEqual([], emqx_authz:get_enabled_authzs()).
t_get_enabled_authzs_some_enabled(_Config) ->
{ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE4]),
{ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE4, ?SOURCE5#{<<"enable">> := false}]),
?assertEqual([postgresql], emqx_authz:get_enabled_authzs()).
t_subscribe_deny_disconnect_publishes_last_will_testament(_Config) ->

5
apps/emqx_dashboard/src/emqx_dashboard_bad_api.erl
View File

@ -21,11 +21,12 @@
-export([init/2]).
init(Req0, State) ->
?SLOG(warning, #{msg => "unexpected_api_access", request => Req0}),
RedactedReq = emqx_utils:redact(Req0),
?SLOG(warning, #{msg => "unexpected_api_access", request => RedactedReq}),
Req = cowboy_req:reply(
404,
#{<<"content-type">> => <<"application/json">>},
<<"{\"code\": \"API_NOT_EXIST\", \"message\": \"Request Path Not Found\"}">>,
Req0
RedactedReq
),
{ok, Req, State}.

4
apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl
View File

@ -58,7 +58,7 @@ prop_coverage_likely_incomplete() ->
{filesize_t(), segsizes_t(), filesize_t()},
?FORALL(
Fragments,
noshrink(segments_t(Filesize, Segsizes, Hole)),
noshrink(segments_t(Filesize, Segsizes, (Hole rem max(Filesize, 1)))),
?TIMEOUT(
?COVERAGE_TIMEOUT,
begin
@ -174,7 +174,7 @@ segment_t(Filesize, Segsizes, Hole) ->
?SUCHTHATMAYBE(
{Offset, Size},
segment_t(Filesize, Segsizes),
(Hole rem Filesize) =< Offset orelse (Hole rem Filesize) > (Offset + Size)
Hole =< Offset orelse Hole > (Offset + Size)
).
segment_t(Filesize, Segsizes) ->

68
apps/emqx_machine/src/emqx_machine_boot.erl
View File

@ -24,6 +24,7 @@
-export([stop_port_apps/0]).
-dialyzer({no_match, [basic_reboot_apps/0]}).
-dialyzer({no_match, [basic_reboot_apps_edition/1]}).
-ifdef(TEST).
-export([sorted_reboot_apps/1, reboot_apps/0]).
@ -126,39 +127,40 @@ reboot_apps() ->
BaseRebootApps ++ ConfigApps.
basic_reboot_apps() ->
CE =
?BASIC_REBOOT_APPS ++
[
emqx_prometheus,
emqx_modules,
emqx_dashboard,
emqx_connector,
emqx_gateway,
emqx_resource,
emqx_rule_engine,
emqx_bridge,
emqx_plugin_libs,
emqx_management,
emqx_retainer,
emqx_exhook,
emqx_authn,
emqx_authz,
emqx_slow_subs,
emqx_auto_subscribe,
emqx_plugins
],
case emqx_release:edition() of
ce ->
CE ++ [emqx_telemetry];
ee ->
CE ++
[
emqx_s3,
emqx_ft,
emqx_eviction_agent,
emqx_node_rebalance
]
end.
?BASIC_REBOOT_APPS ++
[
emqx_prometheus,
emqx_modules,
emqx_dashboard,
emqx_connector,
emqx_gateway,
emqx_resource,
emqx_rule_engine,
emqx_bridge,
emqx_plugin_libs,
emqx_management,
emqx_retainer,
emqx_exhook,
emqx_authn,
emqx_authz,
emqx_slow_subs,
emqx_auto_subscribe,
emqx_plugins
] ++ basic_reboot_apps_edition(emqx_release:edition()).
basic_reboot_apps_edition(ce) ->
[emqx_telemetry];
basic_reboot_apps_edition(ee) ->
[
emqx_license,
emqx_s3,
emqx_ft,
emqx_eviction_agent,
emqx_node_rebalance
];
%% unexcepted edition, should not happen
basic_reboot_apps_edition(_) ->
[].
sorted_reboot_apps() ->
Apps = [{App, app_deps(App)} || App <- reboot_apps()],

2
apps/emqx_telemetry/src/emqx_telemetry.erl
View File

@ -121,7 +121,7 @@ start_reporting() ->
%% @doc Stop the reporting timer.
%% This is an async notification which never fails.
%% This is a no-op in enterprise eidtion.
%% This is a no-op in enterprise edition.
stop_reporting() ->
gen_server:cast(?MODULE, stop_reporting).

17
apps/emqx_telemetry/src/emqx_telemetry_config.erl
View File

@ -32,11 +32,12 @@
-export([
on_server_start/0,
on_server_stop/0,
is_official_version/1
is_official_version/1,
is_official_version/0
]).
is_enabled() ->
IsOfficial = ?MODULE:is_official_version(emqx_release:version()),
IsOfficial = ?MODULE:is_official_version(),
emqx_conf:get([telemetry, enable], IsOfficial).
on_server_start() ->
@ -53,7 +54,9 @@ set_telemetry_status(Status) ->
end.
pre_config_update(_, {set_telemetry_status, Status}, RawConf) ->
{ok, RawConf#{<<"enable">> => Status}}.
{ok, RawConf#{<<"enable">> => Status}};
pre_config_update(_, NewConf, _OldConf) ->
{ok, NewConf}.
post_config_update(
_,
@ -65,6 +68,11 @@ post_config_update(
case Status of
true -> emqx_telemetry:start_reporting();
false -> emqx_telemetry:stop_reporting()
end;
post_config_update(_, _UpdateReq, NewConf, _OldConf, _AppEnvs) ->
case maps:get(enable, NewConf, ?MODULE:is_official_version()) of
true -> emqx_telemetry:start_reporting();
false -> emqx_telemetry:stop_reporting()
end.
cfg_update(Path, Action, Params) ->
@ -74,6 +82,9 @@ cfg_update(Path, Action, Params) ->
#{override_to => cluster}
).
is_official_version() ->
is_official_version(emqx_release:version()).
is_official_version(Version) ->
Pt = "^\\d+\\.\\d+(?:\\.\\d+)?(?:(-(?:alpha|beta|rc)\\.[1-9][0-9]*))?$",
match =:= re:run(Version, Pt, [{capture, none}]).

96
apps/emqx_telemetry/test/emqx_telemetry_api_SUITE.erl
View File

@ -30,6 +30,7 @@ all() ->
init_per_suite(Config) ->
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF),
ok = emqx_common_test_helpers:load_config(emqx_telemetry_schema, ?BASE_CONF),
ok = emqx_mgmt_api_test_util:init_suite(
[emqx_conf, emqx_authn, emqx_authz, emqx_telemetry],
fun set_special_configs/1
@ -52,32 +53,26 @@ end_per_suite(_Config) ->
ok.
init_per_testcase(t_status_non_official, Config) ->
meck:new(emqx_telemetry, [non_strict, passthrough]),
meck:expect(emqx_telemetry, official_version, 1, false),
meck:new(emqx_telemetry_config, [non_strict, passthrough]),
meck:expect(emqx_telemetry_config, is_official_version, 0, false),
%% check non-official telemetry is disable by default
{ok, _} = emqx:update_config([telemetry], #{}),
Config;
init_per_testcase(t_status, Config) ->
meck:new(emqx_telemetry, [non_strict, passthrough]),
meck:expect(emqx_telemetry, enable, fun() -> ok end),
{ok, _, _} =
request(
put,
uri(["telemetry", "status"]),
#{<<"enable">> => true}
),
init_per_testcase(t_status_official, Config) ->
meck:new(emqx_telemetry_config, [non_strict, passthrough]),
meck:expect(emqx_telemetry_config, is_official_version, 0, true),
%% check official telemetry is enable by default
{ok, _} = emqx:update_config([telemetry], #{}),
Config;
init_per_testcase(_TestCase, Config) ->
{ok, _, _} =
request(
put,
uri(["telemetry", "status"]),
#{<<"enable">> => true}
),
%% Force enable telemetry to check data.
{ok, _} = emqx:update_config([telemetry], #{<<"enable">> => true}),
Config.
end_per_testcase(t_status_non_official, _Config) ->
meck:unload(emqx_telemetry);
meck:unload(emqx_telemetry_config);
end_per_testcase(t_status, _Config) ->
meck:unload(emqx_telemetry);
meck:unload(emqx_telemetry_config);
end_per_testcase(_TestCase, _Config) ->
ok.
@ -95,39 +90,50 @@ set_special_configs(_App) ->
%% Tests
%%------------------------------------------------------------------------------
t_status(_) ->
%% official's telemetry is enabled by default
t_status_official(_) ->
check_status(true).
%% non official's telemetry is disabled by default
t_status_non_official(_) ->
check_status(false).
check_status(Default) ->
ct:pal("Check telemetry status:~p~n", [emqx_telemetry_config:is_official_version()]),
?assertEqual(Default, is_telemetry_process_enabled()),
?assertMatch(
{ok, 200, _},
request(
put,
uri(["telemetry", "status"]),
#{<<"enable">> => false}
#{<<"enable">> => (not Default)}
)
),
{ok, 200, Result0} =
request(get, uri(["telemetry", "status"])),
?assertEqual(
#{<<"enable">> => false},
#{<<"enable">> => (not Default)},
emqx_utils_json:decode(Result0)
),
?assertEqual((not Default), is_telemetry_process_enabled()),
?assertMatch(
{ok, 400, _},
request(
put,
uri(["telemetry", "status"]),
#{<<"enable">> => false}
#{<<"enable">> => (not Default)}
)
),
?assertEqual((not Default), is_telemetry_process_enabled()),
?assertMatch(
{ok, 200, _},
request(
put,
uri(["telemetry", "status"]),
#{<<"enable">> => true}
#{<<"enable">> => Default}
)
),
@ -135,30 +141,24 @@ t_status(_) ->
request(get, uri(["telemetry", "status"])),
?assertEqual(
#{<<"enable">> => true},
#{<<"enable">> => Default},
emqx_utils_json:decode(Result1)
),
?assertEqual(Default, is_telemetry_process_enabled()),
?assertMatch(
{ok, 400, _},
request(
put,
uri(["telemetry", "status"]),
#{<<"enable">> => true}
#{<<"enable">> => Default}
)
).
t_status_non_official(_) ->
?assertMatch(
{ok, 200, _},
request(
put,
uri(["telemetry", "status"]),
#{<<"enable">> => false}
)
).
),
?assertEqual(Default, is_telemetry_process_enabled()),
ok.
t_data(_) ->
?assert(is_telemetry_process_enabled()),
{ok, 200, Result} =
request(get, uri(["telemetry", "data"])),
@ -191,3 +191,23 @@ t_data(_) ->
request(get, uri(["telemetry", "data"])),
ok.
%% Support emqx:update_config([telemetry], Conf).
t_conf_update(_) ->
Conf = emqx:get_raw_config([telemetry]),
?assert(is_telemetry_process_enabled()),
{ok, 200, Result1} = request(get, uri(["telemetry", "status"])),
?assertEqual(#{<<"enable">> => true}, emqx_utils_json:decode(Result1)),
{ok, _} = emqx:update_config([telemetry], Conf#{<<"enable">> => false}),
{ok, 200, Result2} = request(get, uri(["telemetry", "status"])),
?assertEqual(#{<<"enable">> => false}, emqx_utils_json:decode(Result2)),
?assertNot(is_telemetry_process_enabled()),
%% reset to true
{ok, _} = emqx:update_config([telemetry], Conf#{<<"enable">> => true}),
?assert(is_telemetry_process_enabled()),
ok.
is_telemetry_process_enabled() ->
%% timer is not undefined.
Timer = element(6, sys:get_state(emqx_telemetry)),
is_reference(Timer).

2
apps/emqx_utils/src/emqx_utils.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_utils, [
{description, "Miscellaneous utilities for EMQX apps"},
% strict semver, bump manually!
{vsn, "5.0.1"},
{vsn, "5.0.2"},
{modules, [
emqx_utils,
emqx_utils_api,

3
apps/emqx_utils/src/emqx_utils.erl
View File

@ -575,6 +575,9 @@ try_to_existing_atom(Convert, Data, Encoding) ->
is_sensitive_key(token) -> true;
is_sensitive_key("token") -> true;
is_sensitive_key(<<"token">>) -> true;
is_sensitive_key(authorization) -> true;
is_sensitive_key("authorization") -> true;
is_sensitive_key(<<"authorization">>) -> true;
is_sensitive_key(password) -> true;
is_sensitive_key("password") -> true;
is_sensitive_key(<<"password">>) -> true;

6
changes/ce/fix-10820.en.md Normal file
View File

@ -0,0 +1,6 @@
In case the cluster updated license before the new node join in. The new node will not apply the updated license.
After this change, the new joined node will use the cluster's license key.
Sometimes the new node must start with a outdated license.
e.g. use emqx-operator deployed and needed to scale up after license expired.
At the time the cluster's license key already updated by API/CLI, but the new node won't use it.

1
changes/ce/fix-10833.en.md Normal file
View File

@ -0,0 +1 @@
Only include enabled authenticators and authorizers in telemetry report, not all of them.

1
changes/ce/fix-10851.en.md Normal file
View File

@ -0,0 +1 @@
Obfuscated sensitive data in the bad API logging.