From 0b07561e9bf598dda7a13ade2323cbf7f4dc671f Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 8 Sep 2022 14:26:49 +0800 Subject: [PATCH 001/165] fix: ensure authentication is array, not struct --- apps/emqx/src/emqx_authentication_config.erl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index 681ed1394..b867800ae 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -64,7 +64,7 @@ pre_config_update(_, UpdateReq, OldConfig) -> try do_pre_config_update(UpdateReq, to_list(OldConfig)) of {error, Reason} -> {error, Reason}; - {ok, NewConfig} -> {ok, return_map(NewConfig)} + {ok, NewConfig} -> {ok, NewConfig} catch throw:Reason -> {error, Reason} @@ -225,9 +225,6 @@ do_check_config(Type, Config, Module) -> throw({bad_authenticator_config, #{type => Type, reason => E}}) end. -return_map([L]) -> L; -return_map(L) -> L. - to_list(undefined) -> []; to_list(M) when M =:= #{} -> []; to_list(M) when is_map(M) -> [M]; From 49829caaa0da32d464bf716f8c4b734b2c562fd5 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 8 Sep 2022 14:27:50 +0800 Subject: [PATCH 002/165] fix: check authn(lazy type) config when boot --- apps/emqx/src/emqx_config.erl | 5 ++--- apps/emqx/src/emqx_schema.erl | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 3d602349d..9bf098ec4 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -414,9 +414,8 @@ check_config(SchemaMod, RawConf) -> check_config(SchemaMod, RawConf, Opts0) -> Opts1 = #{ return_plain => true, - %% TODO: evil, remove, required should be declared in schema - required => false, - format => map + format => map, + check_lazy => true }, Opts = maps:merge(Opts0, Opts1), {AppEnvs, CheckedConf} = diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 35550d4e2..0bc028ec8 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2245,6 +2245,7 @@ validate_alarm_actions(Actions) -> Error -> {error, Error} end. +parse_user_lookup_fun({Fun, _} = Lookup) when is_function(Fun, 3) -> Lookup; parse_user_lookup_fun(StrConf) -> [ModStr, FunStr] = string:tokens(str(StrConf), ": "), Mod = list_to_atom(ModStr), From 0ca89da1a715e382dbd15864087b6e85e1185a54 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 8 Sep 2022 14:44:25 +0800 Subject: [PATCH 003/165] chore: add authn changelog --- CHANGES-5.0.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 13a2fbb30..d2721b5bb 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -1,3 +1,8 @@ +# 5.0.9 +## Bug fixes + +* Ensure authentication type is an array, not struct. [#8923](https://github.com/emqx/emqx/pull/8923) + # 5.0.8 ## Bug fixes @@ -8,6 +13,7 @@ * Speed up dispatching of shared subscription messages in a cluster [#8893](https://github.com/emqx/emqx/pull/8893) * Fix the extra / prefix when CoAP gateway parsing client topics. [#8658](https://github.com/emqx/emqx/pull/8658) * Speed up updating the configuration, When some nodes in the cluster are down. [#8857](https://github.com/emqx/emqx/pull/8857) + * Fix delayed publish inaccurate caused by os time change. [#8926](https://github.com/emqx/emqx/pull/8926) * Fix that EMQX can't start when the retainer is disabled [#8911](https://github.com/emqx/emqx/pull/8911) From 6bd72fe5a14f8e3f7776fa4b50a228513c9e8fd3 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 14 Sep 2022 10:44:36 +0800 Subject: [PATCH 004/165] fix: check authn's mechanism field --- apps/emqx/src/emqx_config.erl | 3 ++- apps/emqx_authn/src/emqx_authn.erl | 4 ++- apps/emqx_authn/src/emqx_authn_app.erl | 36 ++++++++++++++++---------- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 9bf098ec4..c5c67c7a5 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -415,7 +415,8 @@ check_config(SchemaMod, RawConf, Opts0) -> Opts1 = #{ return_plain => true, format => map, - check_lazy => true + %% Don't check lazy types, such as authenticate + check_lazy => false }, Opts = maps:merge(Opts0, Opts1), {AppEnvs, CheckedConf} = diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index 79d269a55..1f986e016 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -70,7 +70,9 @@ do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) -> #{?CONF_NS_BINARY => Config}, Opts#{atom_key => true} ) - end. + end; +do_check_config(_Config, _Opts) -> + throw({invalid_config, "mechanism_field_required"}). atom(Bin) -> try diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index f761bfe33..6823d8694 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -37,8 +37,10 @@ start(_StartType, _StartArgs) -> ok = mria_rlog:wait_for_shards([?AUTH_SHARD], infinity), {ok, Sup} = emqx_authn_sup:start_link(), - ok = initialize(), - {ok, Sup}. + case initialize() of + ok -> {ok, Sup}; + {error, Reason} -> {error, Reason} + end. stop(_State) -> ok = deinitialize(), @@ -49,18 +51,26 @@ stop(_State) -> %%------------------------------------------------------------------------------ initialize() -> - ok = ?AUTHN:register_providers(emqx_authn:providers()), + try + ok = ?AUTHN:register_providers(emqx_authn:providers()), - lists:foreach( - fun({ChainName, RawAuthConfigs}) -> - AuthConfig = emqx_authn:check_configs(RawAuthConfigs), - ?AUTHN:initialize_authentication( - ChainName, - AuthConfig - ) - end, - chain_configs() - ). + lists:foreach( + fun({ChainName, RawAuthConfigs}) -> + AuthConfig = emqx_authn:check_configs(RawAuthConfigs), + ?AUTHN:initialize_authentication( + ChainName, + AuthConfig + ) + end, + chain_configs() + ) + of + ok -> ok + catch + throw:Reason -> + ?SLOG(error, #{msg => "Failed to initialize authentication", reason => Reason}), + {error, {failed_to_initialize_authentication, Reason}} + end. deinitialize() -> ok = ?AUTHN:deregister_providers(provider_types()), From 5056cbebf4c04bd381482b51fc44d68c1db0535c Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 15 Sep 2022 18:04:41 +0800 Subject: [PATCH 005/165] chore: update apps/emqx_authn/src/emqx_authn_app.erl Co-authored-by: Zaiming (Stone) Shi --- apps/emqx_authn/src/emqx_authn_app.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index 6823d8694..5adf067b4 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -68,7 +68,7 @@ initialize() -> ok -> ok catch throw:Reason -> - ?SLOG(error, #{msg => "Failed to initialize authentication", reason => Reason}), + ?SLOG(error, #{msg => "failed_to_initialize_authentication", reason => Reason}), {error, {failed_to_initialize_authentication, Reason}} end. From 595f19cddddffca80c99cdb56064717eb908e2c0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 18 Sep 2022 11:35:39 +0200 Subject: [PATCH 006/165] chore: force fetch upstream tags before appup updates --- scripts/update-appup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh index 1f2b23435..b962c4a9a 100755 --- a/scripts/update-appup.sh +++ b/scripts/update-appup.sh @@ -99,7 +99,7 @@ else pushd "${PREV_DIR_BASE}/${PREV_TAG}" if [ "$NEW_COPY" = 'no' ]; then REMOTE="$(git remote -v | grep "${GIT_REPO}" | head -1 | awk '{print $1}')" - git fetch "$REMOTE" + git fetch "$REMOTE" --tags --force fi git reset --hard git clean -ffdx From c9904e78aaadef80fa0e618ff22b251bad9599c8 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 26 Sep 2022 17:30:54 +0800 Subject: [PATCH 007/165] chore: remove colon port in HTTP-API and CLI In the https://github.com/emqx/emqx/pull/8571, we add a colon before the port. But it was not popular, so we removed ta in this PR --- apps/emqx/src/emqx_listeners.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 67f452e1d..1431538ad 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -521,7 +521,7 @@ merge_default(Options) -> integer() | {tuple(), integer()} | string() | binary() ) -> io_lib:chars(). format_bind(Port) when is_integer(Port) -> - io_lib:format(":~w", [Port]); + io_lib:format("~w", [Port]); %% Print only the port number when bound on all interfaces format_bind({{0, 0, 0, 0}, Port}) -> format_bind(Port); From 5f5ef170ed4c4ccde6e531540a97ecc5cfe5924f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 26 Sep 2022 17:45:34 +0800 Subject: [PATCH 008/165] chore: update changes --- CHANGES-5.0.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 20d972096..c941db91d 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -10,6 +10,9 @@ * Fix GET /listeners API crash When some nodes still in initial configuration. [#9002](https://github.com/emqx/emqx/pull/9002) * Fix empty variable interpolation in authentication and authorization. Placeholders for undefined variables are rendered now as empty strings and do not cause errors anymore. [#8963](https://github.com/emqx/emqx/pull/8963) * Fix the latency statistics error of the slow subscription module when `stats_type` is `internal` or `response`. [#8986](https://github.com/emqx/emqx/pull/8986) +* Change the format of the listener when displaying bind field. + i.e: from `:1883` to `1883`, this fix will affect the output of the HTTP API + and CLI. [#9047](https://github.com/emqx/emqx/pull/9047) # 5.0.8 From 129f09f88bb4081f036264b66677bebb3a2b5071 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 26 Sep 2022 15:23:59 -0300 Subject: [PATCH 009/165] chore: remove extra information from denied lwt log --- apps/emqx/src/emqx_channel.erl | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 8335a2a5d..d1a111dc5 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -2130,11 +2130,7 @@ publish_will_msg(ClientInfo, Msg = #message{topic = Topic}) -> ?tp( warning, last_will_testament_publish_denied, - #{ - client_info => ClientInfo, - topic => Topic, - message => Msg - } + #{topic => Topic} ), ok end. From 4e2aba33aa714066f3d942645d0dd0c6c29e32fe Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 26 Sep 2022 16:26:16 -0300 Subject: [PATCH 010/165] feat(mix): set `snk_kind` from `$kind` to `msg` in Mix build Log before: ``` 2022-09-26T15:51:30.982183-03:00 [warning] '$kind': last_will_testament_publish_denied, clientid: myclient, line: 2132, mfa: emqx_channel:publish_will_msg/2, peername: 127.0.0.1:35228, topic: $SYS/lwt ``` Log after: ``` 2022-09-26T16:25:50.799914-03:00 [warning] clientid: myclient, line: 2132, mfa: emqx_channel:publish_will_msg/2, msg: last_will_testament_publish_denied, peername: 127.0.0.1:47080, topic: $SYS/lwt ``` --- build | 1 + 1 file changed, 1 insertion(+) diff --git a/build b/build index dd2b96604..bd5307014 100755 --- a/build +++ b/build @@ -364,6 +364,7 @@ export_release_vars() { # compiling the project, so that `emqx_release.erl' picks up # `emqx_vsn' as if it was compiled by rebar3. erl_opts+=( "{compile_info,[{emqx_vsn,\"${PKG_VSN}\"}]}" ) + erl_opts+=( "{d,snk_kind,msg}" ) ERL_COMPILER_OPTIONS="[$(join , "${erl_opts[@]}")]" export ERL_COMPILER_OPTIONS From 03eaa07c029e4a6657971eb15aa051190611d0cc Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 27 Sep 2022 12:51:00 +0800 Subject: [PATCH 011/165] chore: add more comments for bind display perference --- CHANGES-5.0.md | 3 --- apps/emqx/src/emqx_listeners.erl | 16 +++++++++------ apps/emqx/test/emqx_listeners_SUITE.erl | 27 +++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index c941db91d..20d972096 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -10,9 +10,6 @@ * Fix GET /listeners API crash When some nodes still in initial configuration. [#9002](https://github.com/emqx/emqx/pull/9002) * Fix empty variable interpolation in authentication and authorization. Placeholders for undefined variables are rendered now as empty strings and do not cause errors anymore. [#8963](https://github.com/emqx/emqx/pull/8963) * Fix the latency statistics error of the slow subscription module when `stats_type` is `internal` or `response`. [#8986](https://github.com/emqx/emqx/pull/8986) -* Change the format of the listener when displaying bind field. - i.e: from `:1883` to `1883`, this fix will affect the output of the HTTP API - and CLI. [#9047](https://github.com/emqx/emqx/pull/9047) # 5.0.8 diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 1431538ad..3d5077675 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -521,12 +521,16 @@ merge_default(Options) -> integer() | {tuple(), integer()} | string() | binary() ) -> io_lib:chars(). format_bind(Port) when is_integer(Port) -> - io_lib:format("~w", [Port]); -%% Print only the port number when bound on all interfaces -format_bind({{0, 0, 0, 0}, Port}) -> - format_bind(Port); -format_bind({{0, 0, 0, 0, 0, 0, 0, 0}, Port}) -> - format_bind(Port); + %% **Note**: + %% 'For TCP, UDP and IP networks, if the host is empty or a literal + %% unspecified IP address, as in ":80", "0.0.0.0:80" or "[::]:80" for + %% TCP and UDP, "", "0.0.0.0" or "::" for IP, the local system is + %% assumed.' + %% + %% Quoted from: https://pkg.go.dev/net + %% Decided to use this format to display the bind for all interfaces and + %% IPv4/IPv6 support + io_lib:format(":~w", [Port]); format_bind({Addr, Port}) when is_list(Addr) -> io_lib:format("~ts:~w", [Addr, Port]); format_bind({Addr, Port}) when is_tuple(Addr), tuple_size(Addr) == 4 -> diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 6ea4d043d..c91f7ba26 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -148,6 +148,33 @@ t_wss_conn(_) -> {ok, Socket} = ssl:connect({127, 0, 0, 1}, 9998, [{verify, verify_none}], 1000), ok = ssl:close(Socket). +t_format_bind(_) -> + ?assertEqual( + ":1883", + lists:flatten(emqx_listeners:format_bind(1883)) + ), + ?assertEqual( + "0.0.0.0:1883", + lists:flatten(emqx_listeners:format_bind({{0, 0, 0, 0}, 1883})) + ), + ?assertEqual( + "[::]:1883", + lists:flatten(emqx_listeners:format_bind({{0, 0, 0, 0, 0, 0, 0, 0}, 1883})) + ), + ?assertEqual( + "127.0.0.1:1883", + lists:flatten(emqx_listeners:format_bind({{127, 0, 0, 1}, 1883})) + ), + ?assertEqual( + ":1883", + lists:flatten(emqx_listeners:format_bind("1883")) + ), + %% due to the emqx_schema:to_ip_port/1, automaticlly add the IPv4 address + ?assertEqual( + "0.0.0.0:1883", + lists:flatten(emqx_listeners:format_bind(":1883")) + ). + render_config_file() -> Path = local_path(["etc", "emqx.conf"]), {ok, Temp} = file:read_file(Path), From 607f68bab1c3ced027b1774cb668235f7018faac Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 27 Sep 2022 13:01:39 +0800 Subject: [PATCH 012/165] chore(schema): don't automaticly add ipv4 address if configured as `:1883` --- apps/emqx/src/emqx_listeners.erl | 2 ++ apps/emqx/src/emqx_schema.erl | 4 ++-- apps/emqx/test/emqx_listeners_SUITE.erl | 3 +-- apps/emqx/test/emqx_shared_sub_SUITE.erl | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 3d5077675..2a809f88d 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -542,6 +542,8 @@ format_bind(Str) when is_list(Str) -> case emqx_schema:to_ip_port(Str) of {ok, {Ip, Port}} -> format_bind({Ip, Port}); + {ok, Port} -> + format_bind(Port); {error, _} -> format_bind(list_to_integer(Str)) end; diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 35550d4e2..dbac27ad7 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -39,7 +39,7 @@ -type comma_separated_binary() :: [binary()]. -type comma_separated_atoms() :: [atom()]. -type bar_separated_list() :: list(). --type ip_port() :: tuple(). +-type ip_port() :: tuple() | integer(). -type cipher() :: map(). -typerefl_from_string({duration/0, emqx_schema, to_duration}). @@ -2169,7 +2169,7 @@ to_bar_separated_list(Str) -> to_ip_port(Str) -> case split_ip_port(Str) of {"", Port} -> - {ok, {{0, 0, 0, 0}, list_to_integer(Port)}}; + {ok, list_to_integer(Port)}; {Ip, Port} -> PortVal = list_to_integer(Port), case inet:parse_address(Ip) of diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index c91f7ba26..5eb216be5 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -169,9 +169,8 @@ t_format_bind(_) -> ":1883", lists:flatten(emqx_listeners:format_bind("1883")) ), - %% due to the emqx_schema:to_ip_port/1, automaticlly add the IPv4 address ?assertEqual( - "0.0.0.0:1883", + ":1883", lists:flatten(emqx_listeners:format_bind(":1883")) ). diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index f53e8f374..8616028ca 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -594,7 +594,7 @@ t_remote(_) -> try {ok, ClientPidLocal} = emqtt:connect(ConnPidLocal), - {ok, ClientPidRemote} = emqtt:connect(ConnPidRemote), + {ok, _ClientPidRemote} = emqtt:connect(ConnPidRemote), emqtt:subscribe(ConnPidRemote, {<<"$share/remote_group/", Topic/binary>>, 0}), From 6836a502ac61f25dd6c11e7b7f97d77bf51a459f Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 27 Sep 2022 13:24:54 +0800 Subject: [PATCH 013/165] fix(delayed): Improve time precision of delayed messages Use milliseconds internally in emqx_delayed to store the publish time, improving precision --- apps/emqx_modules/src/emqx_delayed.erl | 22 ++++++------ apps/emqx_modules/test/emqx_delayed_SUITE.erl | 34 +++++++++++++++++++ 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index ac7f75158..76646bc64 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -91,6 +91,7 @@ -define(SERVER, ?MODULE). -define(MAX_INTERVAL, 4294967). -define(FORMAT_FUN, {?MODULE, format_delayed}). +-define(NOW, erlang:system_time(milli_seconds)). %%-------------------------------------------------------------------- %% Mnesia bootstrap @@ -118,12 +119,13 @@ on_message_publish( {PubAt, Delayed} = case binary_to_integer(Delay) of Interval when Interval < ?MAX_INTERVAL -> - {Interval + erlang:round(Ts / 1000), Interval}; + {Interval * 1000 + Ts, Interval}; Timestamp -> %% Check malicious timestamp? - case (Timestamp - erlang:round(Ts / 1000)) > ?MAX_INTERVAL of + Internal = Timestamp - erlang:round(Ts / 1000), + case Internal > ?MAX_INTERVAL of true -> error(invalid_delayed_timestamp); - false -> {Timestamp, Timestamp - erlang:round(Ts / 1000)} + false -> {Timestamp * 1000, Internal} end end, PubMsg = Msg#message{topic = Topic1}, @@ -189,14 +191,14 @@ format_delayed( WithPayload ) -> PublishTime = to_rfc3339(PublishTimeStamp div 1000), - ExpectTime = to_rfc3339(ExpectTimeStamp), - RemainingTime = ExpectTimeStamp - erlang:system_time(second), + ExpectTime = to_rfc3339(ExpectTimeStamp div 1000), + RemainingTime = ExpectTimeStamp - ?NOW, Result = #{ msgid => emqx_guid:to_hexstr(Id), node => node(), publish_at => PublishTime, delayed_interval => Delayed, - delayed_remaining => RemainingTime, + delayed_remaining => RemainingTime div 1000, expected_at => ExpectTime, topic => Topic, qos => Qos, @@ -296,7 +298,7 @@ handle_cast(Msg, State) -> %% Do Publish... handle_info({timeout, TRef, do_publish}, State = #{publish_timer := TRef}) -> - DeletedKeys = do_publish(mnesia:dirty_first(?TAB), erlang:system_time(seconds)), + DeletedKeys = do_publish(mnesia:dirty_first(?TAB), ?NOW), lists:foreach(fun(Key) -> mria:dirty_delete(?TAB, Key) end, DeletedKeys), {noreply, ensure_publish_timer(State#{publish_timer := undefined, publish_at := 0})}; handle_info(stats, State = #{stats_fun := StatsFun}) -> @@ -347,18 +349,18 @@ ensure_publish_timer(State) -> ensure_publish_timer('$end_of_table', State) -> State#{publish_timer := undefined, publish_at := 0}; ensure_publish_timer({Ts, _Id}, State = #{publish_timer := undefined}) -> - ensure_publish_timer(Ts, erlang:system_time(seconds), State); + ensure_publish_timer(Ts, ?NOW, State); ensure_publish_timer({Ts, _Id}, State = #{publish_timer := TRef, publish_at := PubAt}) when Ts < PubAt -> ok = emqx_misc:cancel_timer(TRef), - ensure_publish_timer(Ts, erlang:system_time(seconds), State); + ensure_publish_timer(Ts, ?NOW, State); ensure_publish_timer(_Key, State) -> State. ensure_publish_timer(Ts, Now, State) -> Interval = max(1, Ts - Now), - TRef = emqx_misc:start_timer(timer:seconds(Interval), do_publish), + TRef = emqx_misc:start_timer(Interval, do_publish), State#{publish_timer := TRef, publish_at := Now + Interval}. do_publish(Key, Now) -> diff --git a/apps/emqx_modules/test/emqx_delayed_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_SUITE.erl index d1af9a064..e5e3db98c 100644 --- a/apps/emqx_modules/test/emqx_delayed_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_SUITE.erl @@ -202,3 +202,37 @@ t_get_basic_usage_info(_Config) -> ), ?assertEqual(#{delayed_message_count => 4}, emqx_delayed:get_basic_usage_info()), ok. + +t_delayed_precision(_) -> + MaxSpan = 1250, + FutureDiff = subscribe_proc(), + DelayedMsg0 = emqx_message:make( + ?MODULE, 1, <<"$delayed/1/delayed/test">>, <<"delayed/test">> + ), + _ = on_message_publish(DelayedMsg0), + ?assert(FutureDiff() =< MaxSpan). + +subscribe_proc() -> + Self = self(), + Ref = erlang:make_ref(), + erlang:spawn(fun() -> + Topic = <<"delayed/+">>, + emqx_broker:subscribe(Topic), + Self ! + {Ref, + receive + {deliver, Topic, Msg} -> + erlang:system_time(milli_seconds) - Msg#message.timestamp + after 2000 -> + 2000 + end}, + emqx_broker:unsubscribe(Topic) + end), + fun() -> + receive + {Ref, Diff} -> + Diff + after 2000 -> + 2000 + end + end. From 56816559d928ce51535bc1ce6b6cf20926e2dbe3 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 27 Sep 2022 13:39:25 +0800 Subject: [PATCH 014/165] chore: bump emqx_modules && update CHANGES --- CHANGES-5.0.md | 6 ++++++ apps/emqx_modules/src/emqx_modules.app.src | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 20d972096..04dffaf8e 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -1,3 +1,9 @@ +# 5.0.10 + +## Enhancements + +* Use milliseconds internally in emqx_delayed to store the publish time, improving precision.[#9060](https://github.com/emqx/emqx/pull/9060) + # 5.0.9 ## Enhancements diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index 2fa38dae3..4a1644152 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.4"}, + {vsn, "5.0.5"}, {modules, []}, {applications, [kernel, stdlib, emqx]}, {mod, {emqx_modules_app, []}}, From 00e4b4da5ad5f19e837cf29b2bebbd15b099e935 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 6 Sep 2022 16:52:22 +0200 Subject: [PATCH 015/165] fix(schema): support hostname.domain:port for mqtt bridge --- apps/emqx/src/emqx_schema.erl | 63 ++++++++++++++----- apps/emqx/test/emqx_schema_tests.erl | 27 ++++++++ apps/emqx/test/emqx_shared_sub_SUITE.erl | 2 +- .../src/mqtt/emqx_connector_mqtt_schema.erl | 2 +- .../src/emqx_dashboard_swagger.erl | 4 +- 5 files changed, 78 insertions(+), 20 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 35550d4e2..59bbf75e9 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -40,6 +40,7 @@ -type comma_separated_atoms() :: [atom()]. -type bar_separated_list() :: list(). -type ip_port() :: tuple(). +-type host_port() :: tuple(). -type cipher() :: map(). -typerefl_from_string({duration/0, emqx_schema, to_duration}). @@ -52,6 +53,7 @@ -typerefl_from_string({comma_separated_binary/0, emqx_schema, to_comma_separated_binary}). -typerefl_from_string({bar_separated_list/0, emqx_schema, to_bar_separated_list}). -typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}). +-typerefl_from_string({host_port/0, emqx_schema, to_host_port}). -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}). -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}). @@ -78,6 +80,7 @@ to_comma_separated_binary/1, to_bar_separated_list/1, to_ip_port/1, + to_host_port/1, to_erl_cipher_suite/1, to_comma_separated_atoms/1 ]). @@ -96,6 +99,7 @@ comma_separated_binary/0, bar_separated_list/0, ip_port/0, + host_port/0, cipher/0, comma_separated_atoms/0 ]). @@ -2167,33 +2171,60 @@ to_bar_separated_list(Str) -> %% - :1883 %% - :::1883 to_ip_port(Str) -> - case split_ip_port(Str) of - {"", Port} -> - {ok, {{0, 0, 0, 0}, list_to_integer(Port)}}; - {Ip, Port} -> + to_host_port(Str, ip_addr). + +%% @doc support the following format: +%% - 127.0.0.1:1883 +%% - ::1:1883 +%% - [::1]:1883 +%% - :1883 +%% - :::1883 +%% - example.com:80 +to_host_port(Str) -> + to_host_port(Str, hostname). + +%% - example.com:80 +to_host_port(Str, IpOrHost) -> + case split_host_port(Str) of + {"", Port} when IpOrHost =:= ip_addr -> + %% this is a local address + {ok, list_to_integer(Port)}; + {"", _Port} -> + %% must specify host part when it's a remote endpoint + {error, bad_host_port}; + {MaybeIp, Port} -> PortVal = list_to_integer(Port), - case inet:parse_address(Ip) of - {ok, R} -> - {ok, {R, PortVal}}; - _ -> + case inet:parse_address(MaybeIp) of + {ok, IpTuple} -> + {ok, {IpTuple, PortVal}}; + _ when IpOrHost =:= hostname -> %% check is a rfc1035's hostname - case inet_parse:domain(Ip) of + case inet_parse:domain(MaybeIp) of true -> - {ok, {Ip, PortVal}}; + {ok, {MaybeIp, PortVal}}; _ -> - {error, Str} - end + {error, bad_hostname} + end; + _ -> + {error, bad_ip_port} end; _ -> - {error, Str} + {error, bad_ip_port} end. -split_ip_port(Str0) -> +split_host_port(Str0) -> Str = re:replace(Str0, " ", "", [{return, list}, global]), case lists:split(string:rchr(Str, $:), Str) of - %% no port + %% no colon {[], Str} -> - error; + try + %% if it's just a port number, then return as-is + _ = list_to_integer(Str), + {"", Str} + catch + _:_ -> + error + end; {IpPlusColon, PortString} -> IpStr0 = lists:droplast(IpPlusColon), case IpStr0 of diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index 9118ac226..fdda9ef44 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -175,3 +175,30 @@ ssl_opts_gc_after_handshake_test_not_rancher_listener_test() -> Checked ), ok. + +to_ip_port_test_() -> + Ip = fun emqx_schema:to_ip_port/1, + Host = fun(Str) -> + case Ip(Str) of + {ok, {_, _} = Res} -> + %% assert + {ok, Res} = emqx_schema:to_host_port(Str); + _ -> + emqx_schema:to_host_port(Str) + end + end, + [ + ?_assertEqual({ok, 80}, Ip("80")), + ?_assertEqual({error, bad_host_port}, Host("80")), + ?_assertEqual({ok, 80}, Ip(":80")), + ?_assertEqual({error, bad_host_port}, Host(":80")), + ?_assertEqual({error, bad_ip_port}, Ip("localhost:80")), + ?_assertEqual({ok, {"localhost", 80}}, Host("localhost:80")), + ?_assertEqual({ok, {"example.com", 80}}, Host("example.com:80")), + ?_assertEqual({ok, {{127, 0, 0, 1}, 80}}, Ip("127.0.0.1:80")), + ?_assertEqual({error, bad_ip_port}, Ip("$:1900")), + ?_assertEqual({error, bad_hostname}, Host("$:1900")), + ?_assertMatch({ok, {_, 1883}}, Ip("[::1]:1883")), + ?_assertMatch({ok, {_, 1883}}, Ip("::1:1883")), + ?_assertMatch({ok, {_, 1883}}, Ip(":::1883")) + ]. diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index f53e8f374..8616028ca 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -594,7 +594,7 @@ t_remote(_) -> try {ok, ClientPidLocal} = emqtt:connect(ConnPidLocal), - {ok, ClientPidRemote} = emqtt:connect(ConnPidRemote), + {ok, _ClientPidRemote} = emqtt:connect(ConnPidRemote), emqtt:subscribe(ConnPidRemote, {<<"$share/remote_group/", Topic/binary>>, 0}), diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index 3e683f5fb..6fa511d00 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -55,7 +55,7 @@ fields("connector") -> )}, {server, sc( - emqx_schema:ip_port(), + emqx_schema:host_port(), #{ required => true, desc => ?DESC("server") diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 52cbc4775..5674d273c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -656,8 +656,8 @@ typename_to_spec("file()", _Mod) -> #{type => string, example => <<"/path/to/file">>}; typename_to_spec("ip_port()", _Mod) -> #{type => string, example => <<"127.0.0.1:80">>}; -typename_to_spec("ip_ports()", _Mod) -> - #{type => string, example => <<"127.0.0.1:80, 127.0.0.2:80">>}; +typename_to_spec("host_port()", _Mod) -> + #{type => string, example => <<"example.host.domain:80">>}; typename_to_spec("url()", _Mod) -> #{type => string, example => <<"http://127.0.0.1">>}; typename_to_spec("connect_timeout()", Mod) -> From cf762671554295ac895602dfca18bf10f38573a2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 24 Sep 2022 23:52:14 +0200 Subject: [PATCH 016/165] docs: add a string value quote example --- rel/emqx_conf.template.en.md | 14 +++++++++++++- rel/emqx_conf.template.zh.md | 14 ++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md index 76d25680b..520ed4578 100644 --- a/rel/emqx_conf.template.en.md +++ b/rel/emqx_conf.template.en.md @@ -143,7 +143,19 @@ to even set complex values from environment variables. For example, this environment variable sets an array value. ``` -export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]" +export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]' +``` + +However this also means a string value should be quoted if it happen to contain special +characters such as `=` and `:`. + +For example, a string value `"localhost:1883"` would be +parsed into object (struct): `{"localhost": 1883}`. + +To keep it as a string, one should quote the value like below: + +``` +EMQX_BRIDGES__MQTT__MYBRIDGE__CONNECTOR_SERVER='"localhost:1883"' ``` ::: tip Tip diff --git a/rel/emqx_conf.template.zh.md b/rel/emqx_conf.template.zh.md index ac4c5ce39..86162a4a9 100644 --- a/rel/emqx_conf.template.zh.md +++ b/rel/emqx_conf.template.zh.md @@ -127,14 +127,24 @@ authentication.1.enable = true 例如 `node.name` 的重载变量名是 `EMQX_NODE__NAME`。 -环境变量的值,是解析成HOCON值的。所以这也使得环境变量可以用来传递复杂数据类型的值。 +环境变量的值,是按 HOCON 值解析的,这也使得环境变量可以用来传递复杂数据类型的值。 例如,下面这个环境变量传入一个数组类型的值。 ``` -export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]" +export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]' ``` +这也意味着有些带特殊字符(例如`:` 和 `=`),则需要用双引号对这个值包起来。 + +例如`localhost:1883` 会被解析成一个结构体 `{"localhost": 1883}`。 +想要把它当字符串使用时,就必需使用引号,如下: + +``` +EMQX_BRIDGES__MQTT__MYBRIDGE__CONNECTOR_SERVER='"localhost:1883"' +``` + + ::: tip Tip 未定义的根路径会被EMQX忽略,例如 `EMQX_UNKNOWN_ROOT__FOOBAR` 这个环境变量会被EMQX忽略, 因为 `UNKNOWN_ROOT` 不是预先定义好的根路径。 From 39ad1723778acc17956517c7c3b8c2496914c18e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 27 Sep 2022 15:19:42 +0800 Subject: [PATCH 017/165] chore: replace ip_port() wuth host_port() for all connectors --- .../emqx_connector/src/emqx_connector_mongo.erl | 2 +- .../emqx_connector/src/emqx_connector_mysql.erl | 2 +- .../emqx_connector/src/emqx_connector_pgsql.erl | 2 +- .../emqx_connector/src/emqx_connector_redis.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_schema.erl | 2 +- apps/emqx_statsd/src/emqx_statsd_schema.erl | 17 +---------------- 6 files changed, 6 insertions(+), 21 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 5b07c5003..a5e2be521 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -345,7 +345,7 @@ init_worker_options([], Acc) -> %% =================================================================== %% Schema funcs -server(type) -> emqx_schema:ip_port(); +server(type) -> emqx_schema:host_port(); server(required) -> true; server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; server(converter) -> fun to_server_raw/1; diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index d6963d04e..79a306b05 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -55,7 +55,7 @@ fields(config) -> emqx_connector_schema_lib:ssl_fields() ++ emqx_connector_schema_lib:prepare_statement_fields(). -server(type) -> emqx_schema:ip_port(); +server(type) -> emqx_schema:host_port(); server(required) -> true; server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; server(converter) -> fun to_server/1; diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 43ec6be8b..f642ba75c 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -58,7 +58,7 @@ fields(config) -> emqx_connector_schema_lib:ssl_fields() ++ emqx_connector_schema_lib:prepare_statement_fields(). -server(type) -> emqx_schema:ip_port(); +server(type) -> emqx_schema:host_port(); server(required) -> true; server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; server(converter) -> fun to_server/1; diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 67310dbac..95677b766 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -97,7 +97,7 @@ fields(sentinel) -> redis_fields() ++ emqx_connector_schema_lib:ssl_fields(). -server(type) -> emqx_schema:ip_port(); +server(type) -> emqx_schema:host_port(); server(required) -> true; server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; server(converter) -> fun to_server_raw/1; diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 8850fa462..e5d6dd52a 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -28,7 +28,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). --type ip_port() :: tuple(). +-type ip_port() :: tuple() | integer(). -type duration() :: non_neg_integer(). -type duration_s() :: non_neg_integer(). -type bytesize() :: pos_integer(). diff --git a/apps/emqx_statsd/src/emqx_statsd_schema.erl b/apps/emqx_statsd/src/emqx_statsd_schema.erl index e8cc32f99..9efde5afc 100644 --- a/apps/emqx_statsd/src/emqx_statsd_schema.erl +++ b/apps/emqx_statsd/src/emqx_statsd_schema.erl @@ -21,8 +21,6 @@ -behaviour(hocon_schema). --export([to_ip_port/1]). - -export([ namespace/0, roots/0, @@ -30,8 +28,6 @@ desc/1 ]). --typerefl_from_string({ip_port/0, emqx_statsd_schema, to_ip_port}). - namespace() -> "statsd". roots() -> ["statsd"]. @@ -55,7 +51,7 @@ fields("statsd") -> desc("statsd") -> ?DESC(statsd); desc(_) -> undefined. -server(type) -> emqx_schema:ip_port(); +server(type) -> emqx_schema:host_port(); server(required) -> true; server(default) -> "127.0.0.1:8125"; server(desc) -> ?DESC(?FUNCTION_NAME); @@ -72,14 +68,3 @@ flush_interval(required) -> true; flush_interval(default) -> "10s"; flush_interval(desc) -> ?DESC(?FUNCTION_NAME); flush_interval(_) -> undefined. - -to_ip_port(Str) -> - case string:tokens(Str, ":") of - [Ip, Port] -> - case inet:parse_address(Ip) of - {ok, R} -> {ok, {R, list_to_integer(Port)}}; - _ -> {error, Str} - end; - _ -> - {error, Str} - end. From 834910b6f6df2c3a360f5eb9c41a02443e907147 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 27 Sep 2022 17:24:44 +0800 Subject: [PATCH 018/165] chore: update app.src --- apps/emqx_connector/src/emqx_connector.app.src | 2 +- apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_statsd/src/emqx_statsd.app.src | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 0734d47b8..06da66398 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "An OTP application"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index adf974243..9d5f85c7b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.5"}, + {vsn, "5.0.6"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx]}, diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 47245c0a2..98dbd6909 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, grpc, emqx, emqx_authn]}, diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src index 21a972266..76b04204b 100644 --- a/apps/emqx_statsd/src/emqx_statsd.app.src +++ b/apps/emqx_statsd/src/emqx_statsd.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_statsd, [ {description, "An OTP application"}, - {vsn, "5.0.1"}, + {vsn, "5.0.2"}, {registered, []}, {mod, {emqx_statsd_app, []}}, {applications, [ From d49675cde5fc50074c8a4ce155f287aaae4a6243 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 28 Sep 2022 21:10:09 +0200 Subject: [PATCH 019/165] chore: supprot DNS discovery test for 4.x --- scripts/test-node-discovery-dns.sh | 62 +++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/scripts/test-node-discovery-dns.sh b/scripts/test-node-discovery-dns.sh index 968269042..e502dbf59 100755 --- a/scripts/test-node-discovery-dns.sh +++ b/scripts/test-node-discovery-dns.sh @@ -1,12 +1,18 @@ #!/usr/bin/env bash -## Test two nodes-cluster discover each other using DNS A records lookup result. +## Test two-nodes cluster discover each other using DNS A records lookup result. set -euo pipefail cd -P -- "$(dirname -- "$0")/.." -IMAGE="${1}" +IMAGE="${1:-}" + +if [ -z "$IMAGE" ]; then + echo "Usage: $0 " + echo "e.g. $0 docker.io/emqx/emqx:5.0.8" + exit 1 +fi NET='test_node_discovery_dns' NODE1='emqx1' @@ -56,7 +62,9 @@ docker run -d -t --name dnsmasq \ --cap-add=NET_ADMIN \ storytel/dnsmasq dnsmasq --no-daemon --log-queries -start_emqx() { +# Node names (the part before '@') should be all the same in the cluster +# e.g. emqx@${IP} +start_emqx_v5() { NAME="$1" IP="$2" DASHBOARD_PORT="$3" @@ -66,6 +74,7 @@ start_emqx() { --ip "$IP" \ --dns "$IP0" \ -p "$DASHBOARD_PORT:18083" \ + -e EMQX_NODE_NAME="emqx@${IP}" \ -e EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug \ -e EMQX_NODE_COOKIE="$COOKIE" \ -e EMQX_cluster__discovery_strategy='dns' \ @@ -74,5 +83,48 @@ start_emqx() { "$IMAGE" } -start_emqx "$NODE1" "$IP1" 18083 -start_emqx "$NODE2" "$IP2" 18084 +## EMQX v4 has different configuration schema: +# EMQX_NODE_NAME="emqx@${IP}": +# This is necessary because 4.x docker entrypoint +# by default uses docker container ID as node name +# (the part before @ of e.g. emqx@172.18.0.101) +# EMQX_cluster__dns__app +# This must be the same as node name in 4.x +# EMQX_cluster__discovery +# This in 5.0 is EMQX_cluster__discovery_strategy +# EMQX_cluster__dns__name +# The DNS domain to lookup for peer nodes +# EMQX_cluster__dns__record_type +# The DNS record type. (only 'a' type is tested) +start_emqx_v4() { + NAME="$1" + IP="$2" + APP_NAME="emqx" + DASHBOARD_PORT="$3" + docker run -d -t \ + --name "$NAME" \ + --net "$NET" \ + --ip "$IP" \ + --dns "$IP0" \ + -p "$DASHBOARD_PORT:18083" \ + -e EMQX_NODE_NAME="${APP_NAME}@${IP}" \ + -e EMQX_LOG__LEVEL=debug \ + -e EMQX_NODE_COOKIE="$COOKIE" \ + -e EMQX_cluster__discovery='dns' \ + -e EMQX_cluster__dns__name="$DOMAIN" \ + -e EMQX_cluster__dns__app="${APP_NAME}" \ + -e EMQX_cluster__dns__record_type="a" \ + "$IMAGE" +} + +case "${IMAGE}" in + *emqx:4.*|*emqx-ee:4.*) + start_emqx_v4 "$NODE1" "$IP1" 18083 + start_emqx_v4 "$NODE2" "$IP2" 18084 + ;; + *) + start_emqx_v5 "$NODE1" "$IP1" 18083 + start_emqx_v5 "$NODE2" "$IP2" 18084 + ;; + +esac From 0c782963c7b1c6ac43564c57f55bf7a4368ad71e Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 29 Sep 2022 15:38:28 +0200 Subject: [PATCH 020/165] chore: don't keep tar.gz in the docker image --- deploy/docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 6c5baa391..50e1187ab 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -16,6 +16,7 @@ RUN export PROFILE=${EMQX_NAME%%-elixir} \ && cd /emqx \ && rm -rf $EMQX_LIB_PATH \ && make $EMQX_NAME1 \ + && rm -f $EMQX_REL_PATH/*.tar.gz \ && mkdir -p /emqx-rel \ && mv $EMQX_REL_PATH /emqx-rel From d1a15f1a431e72bcd72bfbd0becfd4dd0a6e2445 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 30 Sep 2022 15:05:01 +0800 Subject: [PATCH 021/165] chore: improve api descs for banned --- .../i18n/emqx_mgmt_api_banned_i18n.conf | 83 ++++++++----------- 1 file changed, 34 insertions(+), 49 deletions(-) diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf index 686293d0b..6bc736f9e 100644 --- a/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf +++ b/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf @@ -2,112 +2,97 @@ emqx_mgmt_api_banned { list_banned_api { desc { - en: """List banned.""" - zh: """列出黑名单""" - } - label { - en: """List Banned""" - zh: """列出黑名单""" + en: """List all currently banned client IDs, usernames and IP addresses.""" + zh: """列出目前所有被封禁的客户端 ID、用户名和 IP 地址。""" } } create_banned_api { desc { - en: """Create banned.""" - zh: """创建黑名单""" + en: """Add a client ID, username or IP address to the blacklist.""" + zh: """添加一个客户端 ID、用户名或者 IP 地址到黑名单。""" } } create_banned_api_response400 { desc { - en: """Banned already existed, or bad args.""" - zh: """黑名单已存在,或参数格式有错误""" + en: """Bad request, possibly due to wrong parameters or the existence of a banned object.""" + zh: """错误的请求,可能是参数错误或封禁对象已存在等原因。""" } } delete_banned_api { desc { - en: """Delete banned""" - zh: """删除黑名单""" + en: """Remove a client ID, username or IP address from the blacklist.""" + zh: """将一个客户端 ID、用户名或者 IP 地址从黑名单中删除。""" } } delete_banned_api_response404 { desc { - en: """Banned not found. May be the banned time has been exceeded""" - zh: """黑名单未找到,可能为已经超期失效""" - } - } - - create_banned { - desc { - en: """List banned.""" - zh: """列出黑名单""" - } - label { - en: """List Banned""" - zh: """列出黑名单""" + en: """The banned object was not found in the blacklist.""" + zh: """未在黑名单中找到该封禁对象。""" } } as { desc { - en: """Banned type clientid, username, peerhost""" - zh: """黑名单类型,可选 clientid、username、peerhost""" + en: """Ban method, which can be client ID, username or IP address.""" + zh: """封禁方式,可以通过客户端 ID、用户名或者 IP 地址等方式进行封禁。""" } label { - en: """Banned Type""" - zh: """黑名单类型""" + en: """Ban Method""" + zh: """封禁方式""" } } who { desc { - en: """Client info as banned type""" - zh: """设备信息""" + en: """Ban object, specific client ID, username or IP address.""" + zh: """封禁对象,具体的客户端 ID、用户名或者 IP 地址。""" } label { - en: """Banned Info""" - zh: """黑名单信息""" + en: """Ban Object""" + zh: """封禁对象""" } } by { desc { - en: """Commander""" - zh: """黑名单创建者""" + en: """Initiator of the ban.""" + zh: """封禁的发起者。""" } label { - en: """Commander""" - zh: """黑名单创建者""" + en: """Ban Initiator""" + zh: """封禁发起者""" } } reason { desc { - en: """Banned reason""" - zh: """黑名单创建原因""" + en: """Banned reason, record the reason why the current object was banned.""" + zh: """封禁原因,记录当前对象被封禁的原因。""" } label { - en: """Reason""" - zh: """原因""" + en: """Banned Reason""" + zh: """封禁原因""" } } at { desc { - en: """Create banned time, rfc3339, now if not specified""" - zh: """黑名单创建时间,默认为当前""" + en: """The start time of the ban, the format is rfc3339, the default is the time when the operation was initiated.""" + zh: """封禁的起始时间,格式为 rfc3339,默认为发起操作的时间。""" } label { - en: """Create banned time""" - zh: """黑名单创建时间""" + en: """Ban Time""" + zh: """封禁时间""" } } until { desc { - en: """Cancel banned time, rfc3339, now + 5 minute if not specified""" - zh: """黑名单结束时间,默认为创建时间 + 5 分钟""" + en: """The end time of the ban, the formula is rfc3339, the default is the time when the operation was initiated + 5 minutes.""" + zh: """封禁的结束时间,式为 rfc3339,默认为发起操作的时间 + 5 分钟。""" } label { - en: """Cancel banned time""" - zh: """黑名单结束时间""" + en: """Ban End Time""" + zh: """封禁结束时间""" } } } From f08982be93cf3b46d1ac014058409b907412a677 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 30 Sep 2022 15:34:49 +0800 Subject: [PATCH 022/165] chore: improve api desc for alarms --- .../i18n/emqx_mgmt_api_alarms_i18n.conf | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf index c6f518c86..f50e40b62 100644 --- a/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf +++ b/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf @@ -2,78 +2,82 @@ emqx_mgmt_api_alarms { list_alarms_api { desc { - en: """List alarms""" - zh: """列出告警,获取告警列表""" + en: """List currently activated alarms or historical alarms, determined by query parameters.""" + zh: """列出当前激活的告警或历史告警,由查询参数决定。""" } } delete_alarms_api { desc { - en: """Remove all deactivated alarms""" - zh: """删除所有历史告警(非活跃告警)""" + en: """Remove all historical alarms.""" + zh: """删除所有历史告警。""" } } delete_alarms_api_response204 { desc { - en: """Remove all deactivated alarms ok""" - zh: """删除所有历史告警(非活跃告警)成功""" + en: """Historical alarms have been cleared successfully.""" + zh: """历史告警已成功清除。""" } } get_alarms_qs_activated { desc { - en: """Activate alarms, or deactivate alarms. Default is false""" - zh: """活跃中的告警,或历史告警(非活跃告警),默认为 false""" + en: """It is used to specify the alarm type of the query. +When true, it returns the currently activated alarm, +and when it is false, it returns the historical alarm. +The default is false.""" + zh: """用于指定查询的告警类型, +为 true 时返回当前激活的告警,为 false 时返回历史告警,默认为 false。""" } } node { desc { - en: """Alarm in node""" - zh: """告警节点名称""" + en: """The name of the node that triggered this alarm.""" + zh: """触发此告警的节点名称。""" } } name { desc { - en: """Alarm name""" - zh: """告警名称""" + en: """Alarm name, used to distinguish different alarms.""" + zh: """告警名称,用于区分不同的告警。""" } } message { desc { - en: """Alarm readable information""" - zh: """告警信息""" + en: """Alarm message, which describes the alarm content in a human-readable format.""" + zh: """告警消息,以人类可读的方式描述告警内容。""" } } details { desc { - en: """Alarm details information""" - zh: """告警详细信息""" + en: """Alarm details, provides more alarm information, mainly for program processing.""" + zh: """告警详情,提供了更多的告警信息,主要提供给程序处理。""" } } duration { desc { - en: """Alarms duration time; UNIX time stamp, millisecond""" - zh: """告警持续时间,单位:毫秒""" + en: """Indicates how long the alarm has lasted, in milliseconds.""" + zh: """表明告警已经持续了多久,单位:毫秒。""" } } activate_at { desc { - en: """Alarms activate time, RFC 3339""" - zh: """告警开始时间,使用 rfc3339 标准时间格式""" + en: """Alarm start time, using rfc3339 standard time format.""" + zh: """告警开始时间,使用 rfc3339 标准时间格式。""" } } deactivate_at { desc { - en: """Alarms deactivate time, RFC 3339""" - zh: """告警结束时间,使用 rfc3339 标准时间格式""" + en: """Alarm end time, in the rfc3339 standard time format.""" + zh: """告警结束时间,使用 rfc3339 标准时间格式。""" } } From 5fa1e63392a1ac023e12a495cd57e42787cb2761 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 30 Sep 2022 16:01:23 +0800 Subject: [PATCH 023/165] chore(gw): update HTTP API docs for gateway --- .../i18n/emqx_gateway_api_authn_i18n.conf | 39 ++++++++------ .../i18n/emqx_gateway_api_i18n.conf | 42 +++++++++++---- .../i18n/emqx_gateway_api_listeners_i18n.conf | 53 ++++++++++--------- apps/emqx_gateway/src/coap/emqx_coap_api.erl | 3 +- apps/emqx_gateway/src/emqx_gateway_api.erl | 16 ++++-- .../src/emqx_gateway_api_authn.erl | 29 ++++++++-- .../src/emqx_gateway_api_clients.erl | 12 +++++ .../src/emqx_gateway_api_listeners.erl | 33 +++++++++++- .../emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl | 12 +++-- 9 files changed, 173 insertions(+), 66 deletions(-) diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf index 0f04b3938..c5886e8ba 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf @@ -2,70 +2,76 @@ emqx_gateway_api_authn { get_authn { desc { - en: """Get the gateway authentication""" - zh: """获取指定网关认证器""" + en: """Gets the configuration of the specified gateway authenticator.
+Returns 404 when gateway or authentication is not enabled.""" + zh: """获取指定网关认证器的配置 +当网关或认证未启用时,返回 404。""" } } update_authn { desc { - en: """Update authentication for the gateway""" - zh: """更新网关认证器""" + en: """Update the configuration of the specified gateway authenticator, or disable the authenticator.""" + zh: """更新指定网关认证器的配置,或停用认证器。""" } } add_authn { desc { - en: """Add authentication for the gateway""" - zh: """为指定网关新增认证器""" + en: """Enables the authenticator for client authentication for the specified gateway. <\br> +When the authenticator is not configured or turned off, all client connections are assumed to be allowed. <\br> +Note: Adding only one authenticator is supported in the gateway, rather than allowing multiple authenticators to be configured to form an authentication chain as in MQTT.""" + zh: """为指定网关开启认证器实现客户端认证的功能。<\br> +当未配置认证器或关闭认证器时,则认为允许所有客户端的连接。<\br> +注:在网关中仅支持添加一个认证器,而不是像 MQTT 一样允许配置多个认证器构成认证链。""" } } delete_authn { desc { - en: """Remove the gateway authentication""" - zh: """删除指定网关的认证器""" + en: """Delete the authenticator of the specified gateway.""" + zh: """删除指定网关的认证器。""" } } list_users { desc { - en: """Get the users for the authentication""" + en: """Get the users for the authenticator (only supports built_in_database).""" zh: """获取用户列表(仅支持 built_in_database 类型的认证器)""" } } add_user { desc { - en: """Add user for the authentication""" + en: """Add user for the authenticator (only supports built_in_database).""" zh: """添加用户(仅支持 built_in_database 类型的认证器)""" } } get_user { desc { - en: """Get user info from the gateway authentication""" + en: """Get user info from the gateway authenticator (only supports built_in_database)""" zh: """获取用户信息(仅支持 built_in_database 类型的认证器)""" } } update_user { desc { - en: """Update the user info for the gateway authentication""" + en: """Update the user info for the gateway authenticator (only supports built_in_database)""" zh: """更新用户信息(仅支持 built_in_database 类型的认证器)""" } } delete_user { desc { - en: """Delete the user for the gateway authentication""" + en: """Delete the user for the gateway authenticator (only supports built_in_database)""" zh: """删除用户(仅支持 built_in_database 类型的认证器)""" } } import_users { desc { - en: """Import users into the gateway authentication""" + en: """Import users into the gateway authenticator (only supports built_in_database)""" zh: """导入用户(仅支持 built_in_database 类型的认证器)""" } } @@ -79,8 +85,8 @@ emqx_gateway_api_authn { like_user_id { desc { - en: """Fuzzy search by user_id (username or clientid)""" - zh: """用户 ID (username 或 clientid)模糊搜索""" + en: """Fuzzy search using user ID (username or clientid), only supports search by substring.""" + zh: """使用用户 ID (username 或 clientid)模糊搜索,仅支持按子串的方式进行搜索。""" } } @@ -90,5 +96,4 @@ emqx_gateway_api_authn { zh: """是否是超级用户""" } } - } diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf index 5ab9277b2..f7271e6a0 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf @@ -2,15 +2,17 @@ emqx_gateway_api { list_gateway { desc { - en: """Get gateway list""" - zh: """获取网关列表""" + en: """This API returns an overview info for the specified or all gateways. +including current running status, number of connections, listener status, etc.""" + zh: """该接口会返回指定或所有网关的概览状态, +包括当前状态、连接数、监听器状态等。""" } } enable_gateway { desc { - en: """Enable a gateway""" - zh: """启用某网关""" + en: """Enable a gateway by confs.""" + zh: """使用配置启动某一网关。""" } } @@ -23,15 +25,17 @@ emqx_gateway_api { delete_gateway { desc { - en: """Delete/Unload the gateway""" - zh: """删除/禁用某网关""" + en: """Unload the specified gateway""" + zh: """停用指定网关""" } } update_gateway { desc { - en: """Update the gateway configurations/status""" - zh: """更新网关配置或启用状态""" + en: """Update the gateway basic configurations and running status.
+Note: The Authentication and Listener configurations should be updated by other special APIs. """ + zh: """更新指定网关的基础配置、和启用的状态。
+注:认证、和监听器的配置更新需参考对应的 API 接口。""" } } @@ -42,13 +46,33 @@ emqx_gateway_api { } } + gateway_name_in_qs { + desc { + en: """Gateway Name.
+It's enum with `stomp`, `mqttsn`, `coap`, `lwm2m`, `exproto` +""" + zh: """网关名称.
+可取值为 `stomp`、`mqttsn`、`coap`、`lwm2m`、`exproto` +""" + } + } + gateway_status { desc { - en: """Gateway Status""" + en: """Gateway status""" zh: """网关启用状态""" } } + gateway_status_in_qs { + desc { + en: """Filter gateways by status.
+It is enum with `running`, `stopped`, `unloaded`""" + zh: """通过网关状态筛选
+可选值为 `running`、`stopped`、`unloaded`""" + } + } + gateway_created_at { desc { en: """The Gateway created datetime""" diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf index 9c5de67c3..5567a2b5c 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf @@ -2,105 +2,109 @@ emqx_gateway_api_listeners { list_listeners { desc { - en: """Get the gateway listeners""" - zh: """获取网关监听器列表""" + en: """Gets a list of gateway listeners. This interface returns all the configs of the listener (including the authenticator on that listener), as well as the status of that listener running in the cluster.""" + zh: """获取网关监听器列表。该接口会返回监听器所有的配置(包括该监听器上的认证器),同时也会返回该监听器在集群中运行的状态。""" } } add_listener { desc { - en: """Create the gateway listener""" - zh: """为指定网关添加监听器""" + en: """Create the gateway listener.
+Note: For listener types not supported by a gateway, this API returns `400: BAD_REQUEST`.""" + zh: """为指定网关添加监听器。
+注:对于某网关不支持的监听器类型,该接口会返回 `400: BAD_REQUEST`。""" } } get_listener { desc { - en: """Get the gateway listener configurations""" - zh: """获取指定监听器信息""" + en: """Get the gateway listener configs""" + zh: """获取指定网关监听器的配置。""" } } delete_listener { desc { - en: """Delete the gateway listener""" - zh: """删除监听器""" + en: """Delete the gateway listener. All connected clients under the deleted listener will be disconnected.""" + zh: """删除指定监听器。被删除的监听器下所有已连接的客户端都会离线。""" } } update_listener { desc { - en: """Update the gateway listener""" - zh: """更新监听器""" + en: """Update the gateway listener. The listener being updated performs a restart and all clients connected to that listener will be disconnected.""" + zh: """更新某网关监听器的配置。被更新的监听器会执行重启,所有已连接到该监听器上的客户端都会被断开。""" } } get_listener_authn { desc { - en: """Get the listener's authentication info""" - zh: """获取监听器的认证器信息""" + en: """Get the listener's authenticator configs.""" + zh: """获取监听器的认证器配置。""" } } add_listener_authn { desc { - en: """Add authentication for the listener""" - zh: """为指定监听器添加认证器""" + en: """Enable authenticator for specified listener for client authentication.
+When authenticator is enabled for a listener, all clients connected to that listener will use that authenticator for authentication.""" + zh: """为指定监听器开启认证器以实现客户端认证的能力。
+当某一监听器开启认证后,所有连接到该监听器的客户端会使用该认证器进行认证。""" } } update_listener_authn { desc { - en: """Update authentication for the listener""" - zh: """更新指定监听上的认证器配置""" + en: """Update authenticator configs for the listener, or disable/enable it.""" + zh: """更新指定监听器的认证器配置,或停用/启用该认证器。""" } } delete_listener_authn { desc { - en: """Remove authentication for the listener""" - zh: """为指定监听器移除认证器""" + en: """Remove authenticator for the listener.""" + zh: """移除指定监听器的认证器。""" } } list_users { desc { - en: """Get the users for the authentication""" + en: """Get the users for the authenticator (only supports built_in_database)""" zh: """获取用户列表(仅支持 built_in_database 类型的认证器)""" } } add_user { desc { - en: """Add user for the authentication""" + en: """Add user for the authenticator (only supports built_in_database)""" zh: """添加用户(仅支持 built_in_database 类型的认证器)""" } } get_user { desc { - en: """Get user info from the gateway authentication""" + en: """Get user info from the gateway authenticator (only supports built_in_database)""" zh: """获取用户信息(仅支持 built_in_database 类型的认证器)""" } } update_user { desc { - en: """Update the user info for the gateway authentication""" + en: """Update the user info for the gateway authenticator (only supports built_in_database)""" zh: """更新用户信息(仅支持 built_in_database 类型的认证器)""" } } delete_user { desc { - en: """Delete the user for the gateway authentication""" + en: """Delete the user for the gateway authenticator (only supports built_in_database)""" zh: """删除用户(仅支持 built_in_database 类型的认证器)""" } } import_users { desc { - en: """Import users into the gateway authentication""" + en: """Import users into the gateway authenticator (only supports built_in_database)""" zh: """导入用户(仅支持 built_in_database 类型的认证器)""" } } @@ -139,5 +143,4 @@ emqx_gateway_api_listeners { zh: """当前连接数""" } } - } diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl index 5e94934da..27cb23ac8 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl @@ -48,8 +48,9 @@ schema(?PREFIX ++ "/request") -> #{ operationId => request, post => #{ - tags => [<<"CoAP">>], + tags => [<<"Gateways">>], desc => ?DESC(send_coap_request), + summary => <<"Send a Request to a Client">>, parameters => request_parameters(), requestBody => request_body(), responses => #{ diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 5ae8fe1e7..00d3d4b17 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -164,7 +164,9 @@ schema("/gateways") -> 'operationId' => gateway, get => #{ + tags => [<<"Gateways">>], desc => ?DESC(list_gateway), + summary => <<"List All Gateways">>, parameters => params_gateway_status_in_qs(), responses => ?STANDARD_RESP( @@ -178,7 +180,9 @@ schema("/gateways") -> }, post => #{ + tags => [<<"Gateways">>], desc => ?DESC(enable_gateway), + summary => <<"Enable a Gateway">>, %% TODO: distinguish create & response swagger schema 'requestBody' => schema_gateways_conf(), responses => @@ -190,21 +194,27 @@ schema("/gateways/:name") -> 'operationId' => gateway_insta, get => #{ + tags => [<<"Gateways">>], desc => ?DESC(get_gateway), + summary => <<"Get the Gateway">>, parameters => params_gateway_name_in_path(), responses => ?STANDARD_RESP(#{200 => schema_gateways_conf()}) }, delete => #{ + tags => [<<"Gateways">>], desc => ?DESC(delete_gateway), + summary => <<"Unload the gateway">>, parameters => params_gateway_name_in_path(), responses => ?STANDARD_RESP(#{204 => <<"Deleted">>}) }, put => #{ + tags => [<<"Gateways">>], desc => ?DESC(update_gateway), + summary => <<"Update the gateway confs">>, parameters => params_gateway_name_in_path(), 'requestBody' => schema_update_gateways_conf(), responses => @@ -224,8 +234,8 @@ params_gateway_name_in_path() -> binary(), #{ in => path, - desc => ?DESC(gateway_name), - example => <<"">> + desc => ?DESC(gateway_name_in_qs), + example => <<"stomp">> } )} ]. @@ -239,7 +249,7 @@ params_gateway_status_in_qs() -> #{ in => query, required => false, - desc => ?DESC(gateway_status), + desc => ?DESC(gateway_status_in_qs), example => <<"">> } )} diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl index 6fd073a3b..e915e0fa2 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl @@ -75,6 +75,7 @@ authn(get, #{bindings := #{name := Name0}}) -> Authn -> {200, Authn} catch error:{config_not_found, _} -> + %% FIXME: should return 404? {204} end end); @@ -181,19 +182,23 @@ schema("/gateways/:name/authentication") -> 'operationId' => authn, get => #{ + tag => [<<"Gateways">>], desc => ?DESC(get_authn), + summary => <<"Get Authenticator Configuration">>, parameters => params_gateway_name_in_path(), responses => ?STANDARD_RESP( #{ 200 => schema_authn(), - 204 => <<"Authentication does not initiated">> + 204 => <<"Authenticator doesn't initiated">> } ) }, put => #{ + tag => [<<"Gateways">>], desc => ?DESC(update_authn), + summary => <<"Update Authenticator Configuration">>, parameters => params_gateway_name_in_path(), 'requestBody' => schema_authn(), responses => @@ -201,7 +206,9 @@ schema("/gateways/:name/authentication") -> }, post => #{ + tag => [<<"Gateways">>], desc => ?DESC(add_authn), + summary => <<"Create an Authenticator for a Gateway">>, parameters => params_gateway_name_in_path(), 'requestBody' => schema_authn(), responses => @@ -209,7 +216,9 @@ schema("/gateways/:name/authentication") -> }, delete => #{ + tag => [<<"Gateways">>], desc => ?DESC(delete_authn), + summary => <<"Delete the Gateway Authenticator">>, parameters => params_gateway_name_in_path(), responses => ?STANDARD_RESP(#{204 => <<"Deleted">>}) @@ -220,7 +229,9 @@ schema("/gateways/:name/authentication/users") -> 'operationId' => users, get => #{ + tag => [<<"Gateways">>], desc => ?DESC(list_users), + summary => <<"List users for a Gateway Authenticator">>, parameters => params_gateway_name_in_path() ++ params_paging_in_qs() ++ params_fuzzy_in_qs(), @@ -236,7 +247,9 @@ schema("/gateways/:name/authentication/users") -> }, post => #{ + tag => [<<"Gateways">>], desc => ?DESC(add_user), + summary => <<"Add User for a Gateway Authenticator">>, parameters => params_gateway_name_in_path(), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( ref(emqx_authn_api, request_user_create), @@ -258,7 +271,9 @@ schema("/gateways/:name/authentication/users/:uid") -> 'operationId' => users_insta, get => #{ + tag => [<<"Gateways">>], desc => ?DESC(get_user), + summary => <<"Get User Info for a Gateway Authenticator">>, parameters => params_gateway_name_in_path() ++ params_userid_in_path(), responses => @@ -273,7 +288,9 @@ schema("/gateways/:name/authentication/users/:uid") -> }, put => #{ + tag => [<<"Gateways">>], desc => ?DESC(update_user), + summary => <<"Update User Info for a Gateway Authenticator">>, parameters => params_gateway_name_in_path() ++ params_userid_in_path(), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( @@ -292,7 +309,9 @@ schema("/gateways/:name/authentication/users/:uid") -> }, delete => #{ + tag => [<<"Gateways">>], desc => ?DESC(delete_user), + summary => <<"Delete User for a Gateway Authenticator">>, parameters => params_gateway_name_in_path() ++ params_userid_in_path(), responses => @@ -311,8 +330,8 @@ params_gateway_name_in_path() -> binary(), #{ in => path, - desc => ?DESC(emqx_gateway_api, gateway_name), - example => <<"">> + desc => ?DESC(emqx_gateway_api, gateway_name_in_qs), + example => <<"stomp">> } )} ]. @@ -325,7 +344,7 @@ params_userid_in_path() -> #{ in => path, desc => ?DESC(user_id), - example => <<"">> + example => <<"test_username">> } )} ]. @@ -343,7 +362,7 @@ params_fuzzy_in_qs() -> in => query, required => false, desc => ?DESC(like_user_id), - example => <<"username">> + example => <<"test_">> } )}, {is_superuser, diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index b7cf9fc64..64ec46105 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -467,7 +467,9 @@ schema("/gateways/:name/clients") -> 'operationId' => clients, get => #{ + tag => [<<"Gateways">>], desc => ?DESC(list_clients), + summary => <<"List Gateway's Clients">>, parameters => params_client_query(), responses => ?STANDARD_RESP(#{200 => schema_client_list()}) @@ -478,14 +480,18 @@ schema("/gateways/:name/clients/:clientid") -> 'operationId' => clients_insta, get => #{ + tag => [<<"Gateways">>], desc => ?DESC(get_client), + summary => <<"Get Client Info">>, parameters => params_client_insta(), responses => ?STANDARD_RESP(#{200 => schema_client()}) }, delete => #{ + tag => [<<"Gateways">>], desc => ?DESC(kick_client), + summary => <<"Kick out Client">>, parameters => params_client_insta(), responses => ?STANDARD_RESP(#{204 => <<"Kicked">>}) @@ -496,7 +502,9 @@ schema("/gateways/:name/clients/:clientid/subscriptions") -> 'operationId' => subscriptions, get => #{ + tag => [<<"Gateways">>], desc => ?DESC(list_subscriptions), + summary => <<"List Client's Subscription">>, parameters => params_client_insta(), responses => ?STANDARD_RESP( @@ -510,7 +518,9 @@ schema("/gateways/:name/clients/:clientid/subscriptions") -> }, post => #{ + tag => [<<"Gateways">>], desc => ?DESC(add_subscription), + summary => <<"Add Subscription for Client">>, parameters => params_client_insta(), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( ref(subscription), @@ -532,7 +542,9 @@ schema("/gateways/:name/clients/:clientid/subscriptions/:topic") -> 'operationId' => subscriptions, delete => #{ + tag => [<<"Gateways">>], desc => ?DESC(delete_subscription), + summary => <<"Delete Client's Subscription">>, parameters => params_topic_name_in_path() ++ params_client_insta(), responses => ?STANDARD_RESP(#{204 => <<"Unsubscribed">>}) diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 92903ec35..a2d55edb9 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -358,7 +358,9 @@ schema("/gateways/:name/listeners") -> 'operationId' => listeners, get => #{ + tags => [<<"Gateways">>], desc => ?DESC(list_listeners), + summary => <<"List All Listener">>, parameters => params_gateway_name_in_path(), responses => ?STANDARD_RESP( @@ -372,7 +374,9 @@ schema("/gateways/:name/listeners") -> }, post => #{ + tags => [<<"Gateways">>], desc => ?DESC(add_listener), + summary => <<"Add a Listener">>, parameters => params_gateway_name_in_path(), %% XXX: How to distinguish the different listener supported by %% different types of gateways? @@ -396,7 +400,9 @@ schema("/gateways/:name/listeners/:id") -> 'operationId' => listeners_insta, get => #{ + tags => [<<"Gateways">>], desc => ?DESC(get_listener), + summary => <<"Get the Listener Configs">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), responses => @@ -411,7 +417,9 @@ schema("/gateways/:name/listeners/:id") -> }, delete => #{ + tags => [<<"Gateways">>], desc => ?DESC(delete_listener), + summary => <<"Delete the Listener">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), responses => @@ -419,7 +427,9 @@ schema("/gateways/:name/listeners/:id") -> }, put => #{ + tags => [<<"Gateways">>], desc => ?DESC(update_listener), + summary => <<"Update the Listener Configs">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( @@ -442,7 +452,9 @@ schema("/gateways/:name/listeners/:id/authentication") -> 'operationId' => listeners_insta_authn, get => #{ + tags => [<<"Gateways">>], desc => ?DESC(get_listener_authn), + summary => <<"Get the Listener's Authenticator">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), responses => @@ -455,7 +467,9 @@ schema("/gateways/:name/listeners/:id/authentication") -> }, post => #{ + tags => [<<"Gateways">>], desc => ?DESC(add_listener_authn), + summary => <<"Create an Authenticator for a Listener">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), 'requestBody' => schema_authn(), @@ -464,7 +478,9 @@ schema("/gateways/:name/listeners/:id/authentication") -> }, put => #{ + tags => [<<"Gateways">>], desc => ?DESC(update_listener_authn), + summary => <<"Update the Listener Authenticator configs">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), 'requestBody' => schema_authn(), @@ -473,7 +489,9 @@ schema("/gateways/:name/listeners/:id/authentication") -> }, delete => #{ + tags => [<<"Gateways">>], desc => ?DESC(delete_listener_authn), + summary => <<"Delete the Listener's Authenticator">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), responses => @@ -485,7 +503,9 @@ schema("/gateways/:name/listeners/:id/authentication/users") -> 'operationId' => users, get => #{ + tags => [<<"Gateways">>], desc => ?DESC(list_users), + summary => <<"List Authenticator's Users">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path() ++ params_paging_in_qs(), @@ -501,7 +521,9 @@ schema("/gateways/:name/listeners/:id/authentication/users") -> }, post => #{ + tags => [<<"Gateways">>], desc => ?DESC(add_user), + summary => <<"Add User for an Authenticator">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( @@ -524,7 +546,9 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") -> 'operationId' => users_insta, get => #{ + tags => [<<"Gateways">>], desc => ?DESC(get_user), + summary => <<"Get User Info">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path() ++ params_userid_in_path(), @@ -540,7 +564,9 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") -> }, put => #{ + tags => [<<"Gateways">>], desc => ?DESC(update_user), + summary => <<"Update User Info">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path() ++ params_userid_in_path(), @@ -560,7 +586,9 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") -> }, delete => #{ + tags => [<<"Gateways">>], desc => ?DESC(delete_user), + summary => <<"Delete User">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path() ++ params_userid_in_path(), @@ -570,6 +598,7 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") -> }; schema(Path) -> emqx_gateway_utils:make_compatible_schema(Path, fun schema/1). + %%-------------------------------------------------------------------- %% params defines @@ -580,8 +609,8 @@ params_gateway_name_in_path() -> binary(), #{ in => path, - desc => ?DESC(emqx_gateway_api, gateway_name), - example => <<"">> + desc => ?DESC(emqx_gateway_api, gateway_name_in_qs), + example => <<"stomp">> } )} ]. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl index 1aa0bac93..b563d9ea6 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl @@ -45,8 +45,9 @@ schema(?PATH("/lookup")) -> #{ 'operationId' => lookup, get => #{ - tags => [<<"LwM2M">>], + tags => [<<"Gateways">>], desc => ?DESC(lookup_resource), + summary => <<"List Client's Resources">>, parameters => [ {clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})}, {path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})}, @@ -69,8 +70,9 @@ schema(?PATH("/observe")) -> #{ 'operationId' => observe, post => #{ - tags => [<<"LwM2M">>], + tags => [<<"Gateways">>], desc => ?DESC(observe_resource), + summary => <<"Observe a Resource">>, parameters => [ {clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})}, {path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})}, @@ -87,8 +89,9 @@ schema(?PATH("/read")) -> #{ 'operationId' => read, post => #{ - tags => [<<"LwM2M">>], + tags => [<<"Gateways">>], desc => ?DESC(read_resource), + summary => <<"Read Value from a Resource Path">>, parameters => [ {clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})}, {path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})} @@ -104,7 +107,8 @@ schema(?PATH("/write")) -> 'operationId' => write, post => #{ desc => ?DESC(write_resource), - tags => [<<"LwM2M">>], + tags => [<<"Gateways">>], + summary => <<"Write a Value to Resource Path">>, parameters => [ {clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})}, {path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})}, From 0975bf21a99f444263d9070c2dd797885bad5c81 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 30 Sep 2022 17:55:12 +0800 Subject: [PATCH 024/165] docs: improve api desc for statsd --- .../i18n/emqx_mgmt_api_alarms_i18n.conf | 2 +- .../i18n/emqx_statsd_schema_i18n.conf | 34 +++++++++++++------ apps/emqx_statsd/src/emqx_statsd_api.erl | 5 +-- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf index f50e40b62..8932df048 100644 --- a/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf +++ b/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf @@ -76,7 +76,7 @@ The default is false.""" deactivate_at { desc { - en: """Alarm end time, in the rfc3339 standard time format.""" + en: """Alarm end time, using rfc3339 standard time format.""" zh: """告警结束时间,使用 rfc3339 标准时间格式。""" } } diff --git a/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf b/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf index 4ccad1682..9c6eb5afb 100644 --- a/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf +++ b/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf @@ -1,9 +1,23 @@ emqx_statsd_schema { + get_statsd_config_api { + desc { + en: """List the configuration of StatsD metrics collection and push service.""" + zh: """列出 StatsD 指标采集和推送服务的的配置。""" + } + } + + update_statsd_config_api { + desc { + en: """Update the configuration of StatsD metrics collection and push service.""" + zh: """更新 StatsD 指标采集和推送服务的配置。""" + } + } + statsd { desc { - en: """Settings for reporting metrics to StatsD""" - zh: """StatsD 监控数据推送""" + en: """StatsD metrics collection and push configuration.""" + zh: """StatsD 指标采集与推送配置。""" } label { en: """StatsD""" @@ -13,29 +27,29 @@ emqx_statsd_schema { server { desc { - en: """URL of StatsD server""" - zh: """StatsD 服务器地址""" + en: """StatsD server address.""" + zh: """StatsD 服务器地址。""" } } sample_interval { desc { - en: """Data collection interval.""" - zh: """数据收集间隔""" + en: """The sampling interval for metrics.""" + zh: """指标的采样间隔。""" } } flush_interval { desc { - en: """Data reporting interval.""" - zh: """数据推送间隔""" + en: """The push interval for metrics.""" + zh: """指标的推送间隔。""" } } enable { desc { - en: """Turn StatsD data pushing on or off""" - zh: """开启或关闭 StatsD 数据推送""" + en: """Enable or disable StatsD metrics collection and push service.""" + zh: """启用或禁用 StatsD 指标采集和推送服务。""" } } } diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl index ee6007d7d..2f2e42303 100644 --- a/apps/emqx_statsd/src/emqx_statsd_api.erl +++ b/apps/emqx_statsd/src/emqx_statsd_api.erl @@ -20,6 +20,7 @@ -include("emqx_statsd.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). -import(hoconsc, [mk/2, ref/2]). @@ -48,14 +49,14 @@ schema("/statsd") -> 'operationId' => statsd, get => #{ - description => <<"Get statsd config">>, + description => ?DESC(get_statsd_config_api), tags => ?API_TAG_STATSD, responses => #{200 => statsd_config_schema()} }, put => #{ - description => <<"Set statsd config">>, + description => ?DESC(update_statsd_config_api), tags => ?API_TAG_STATSD, 'requestBody' => statsd_config_schema(), responses => From d17c473dc2951c923882f7eb7ab35d1304223995 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 30 Sep 2022 18:57:08 +0800 Subject: [PATCH 025/165] chore(i18n): fix the bad desc for statsd --- .../i18n/emqx_mgmt_api_alarms_i18n.conf | 6 +++--- apps/emqx_statsd/i18n/emqx_statsd_api_i18n.conf | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 apps/emqx_statsd/i18n/emqx_statsd_api_i18n.conf diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf index 8932df048..0ab09520e 100644 --- a/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf +++ b/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf @@ -23,9 +23,9 @@ emqx_mgmt_api_alarms { get_alarms_qs_activated { desc { - en: """It is used to specify the alarm type of the query. -When true, it returns the currently activated alarm, -and when it is false, it returns the historical alarm. + en: """It is used to specify the alarm type of the query. +When true, it returns the currently activated alarm, +and when it is false, it returns the historical alarm. The default is false.""" zh: """用于指定查询的告警类型, 为 true 时返回当前激活的告警,为 false 时返回历史告警,默认为 false。""" diff --git a/apps/emqx_statsd/i18n/emqx_statsd_api_i18n.conf b/apps/emqx_statsd/i18n/emqx_statsd_api_i18n.conf new file mode 100644 index 000000000..2721188bd --- /dev/null +++ b/apps/emqx_statsd/i18n/emqx_statsd_api_i18n.conf @@ -0,0 +1,16 @@ +emqx_statsd_api { + + get_statsd_config_api { + desc { + en: """List the configuration of StatsD metrics collection and push service.""" + zh: """列出 StatsD 指标采集和推送服务的的配置。""" + } + } + + update_statsd_config_api { + desc { + en: """Update the configuration of StatsD metrics collection and push service.""" + zh: """更新 StatsD 指标采集和推送服务的配置。""" + } + } +} From 8292572da62831b35efcdda501d023d3c54f0b9a Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 30 Sep 2022 23:02:28 +0800 Subject: [PATCH 026/165] chore: apply suggestions from code review Co-authored-by: Thales Macedo Garitezi --- apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf | 2 +- apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf | 4 ++-- apps/emqx_gateway/src/emqx_gateway_api_listeners.erl | 2 +- apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf index c5886e8ba..c1f74e531 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf @@ -36,7 +36,7 @@ Note: Adding only one authenticator is supported in the gateway, rather than all list_users { desc { - en: """Get the users for the authenticator (only supports built_in_database).""" + en: """Get the users for the authenticator (only supported by built_in_database).""" zh: """获取用户列表(仅支持 built_in_database 类型的认证器)""" } } diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf index 5567a2b5c..0b7c93afe 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf @@ -47,7 +47,7 @@ Note: For listener types not supported by a gateway, this API returns `400: BAD_ add_listener_authn { desc { en: """Enable authenticator for specified listener for client authentication.
-When authenticator is enabled for a listener, all clients connected to that listener will use that authenticator for authentication.""" +When authenticator is enabled for a listener, all clients connecting to that listener will use that authenticator for authentication.""" zh: """为指定监听器开启认证器以实现客户端认证的能力。
当某一监听器开启认证后,所有连接到该监听器的客户端会使用该认证器进行认证。""" } @@ -69,7 +69,7 @@ When authenticator is enabled for a listener, all clients connected to that list list_users { desc { - en: """Get the users for the authenticator (only supports built_in_database)""" + en: """Get the users for the authenticator (only supported by built_in_database)""" zh: """获取用户列表(仅支持 built_in_database 类型的认证器)""" } } diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index a2d55edb9..259ca8b36 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -360,7 +360,7 @@ schema("/gateways/:name/listeners") -> #{ tags => [<<"Gateways">>], desc => ?DESC(list_listeners), - summary => <<"List All Listener">>, + summary => <<"List All Listeners">>, parameters => params_gateway_name_in_path(), responses => ?STANDARD_RESP( diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf index 6bc736f9e..3045cb293 100644 --- a/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf +++ b/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf @@ -67,11 +67,11 @@ emqx_mgmt_api_banned { } reason { desc { - en: """Banned reason, record the reason why the current object was banned.""" + en: """Ban reason, record the reason why the current object was banned.""" zh: """封禁原因,记录当前对象被封禁的原因。""" } label { - en: """Banned Reason""" + en: """Ban Reason""" zh: """封禁原因""" } } @@ -87,7 +87,7 @@ emqx_mgmt_api_banned { } until { desc { - en: """The end time of the ban, the formula is rfc3339, the default is the time when the operation was initiated + 5 minutes.""" + en: """The end time of the ban, the format is rfc3339, the default is the time when the operation was initiated + 5 minutes.""" zh: """封禁的结束时间,式为 rfc3339,默认为发起操作的时间 + 5 分钟。""" } label { From 9c6c13789e8561d873f99aae72d33d65a02eecd9 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 30 Sep 2022 23:03:01 +0800 Subject: [PATCH 027/165] chore: update apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf Co-authored-by: Thales Macedo Garitezi --- apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf index c1f74e531..b7a65bee1 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf @@ -20,7 +20,7 @@ Returns 404 when gateway or authentication is not enabled.""" desc { en: """Enables the authenticator for client authentication for the specified gateway. <\br> When the authenticator is not configured or turned off, all client connections are assumed to be allowed. <\br> -Note: Adding only one authenticator is supported in the gateway, rather than allowing multiple authenticators to be configured to form an authentication chain as in MQTT.""" +Note: Only one authenticator is allowed to be enabled at a time in the gateway, rather than allowing multiple authenticators to be configured to form an authentication chain as in MQTT.""" zh: """为指定网关开启认证器实现客户端认证的功能。<\br> 当未配置认证器或关闭认证器时,则认为允许所有客户端的连接。<\br> 注:在网关中仅支持添加一个认证器,而不是像 MQTT 一样允许配置多个认证器构成认证链。""" From a84c6f938c1bf31d5132183a6c6ffd0910c863d4 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 30 Sep 2022 23:04:26 +0800 Subject: [PATCH 028/165] chore(swagger): caption tags name --- apps/emqx_authn/src/emqx_authn_api.erl | 2 +- .../src/emqx_auto_subscribe_api.erl | 4 ++-- .../src/emqx_dashboard_error_code_api.erl | 4 ++-- apps/emqx_exhook/src/emqx_exhook_api.erl | 2 +- .../src/emqx_gateway_api_authn.erl | 18 +++++++++--------- .../src/emqx_gateway_api_clients.erl | 12 ++++++------ apps/emqx_management/src/emqx_mgmt_api_app.erl | 2 +- apps/emqx_management/src/emqx_mgmt_api_sys.erl | 2 +- apps/emqx_slow_subs/src/emqx_slow_subs_api.erl | 2 +- 9 files changed, 24 insertions(+), 24 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index c7f8eb824..6b5a8c5cd 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -34,7 +34,7 @@ % Swagger -define(API_TAGS_GLOBAL, [<<"Authentication">>]). --define(API_TAGS_SINGLE, [<<"Listener authentication">>]). +-define(API_TAGS_SINGLE, [<<"Listener Authentication">>]). -export([ api_spec/0, diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl index 4edb709b9..4d48bfced 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl @@ -44,14 +44,14 @@ schema("/mqtt/auto_subscribe") -> 'operationId' => auto_subscribe, get => #{ description => ?DESC(list_auto_subscribe_api), - tags => [<<"Auto subscribe">>], + tags => [<<"Auto Subscribe">>], responses => #{ 200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe") } }, put => #{ description => ?DESC(update_auto_subscribe_api), - tags => [<<"Auto subscribe">>], + tags => [<<"Auto Subscribe">>], 'requestBody' => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"), responses => #{ 200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"), diff --git a/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl index 139567828..2605ad91e 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl @@ -51,7 +51,7 @@ schema("/error_codes") -> get => #{ security => [], description => <<"API Error Codes">>, - tags => [<<"Error codes">>], + tags => [<<"Error Codes">>], responses => #{ 200 => hoconsc:array(hoconsc:ref(?MODULE, error_code)) } @@ -63,7 +63,7 @@ schema("/error_codes/:code") -> get => #{ security => [], description => <<"API Error Codes">>, - tags => [<<"Error codes">>], + tags => [<<"Error Codes">>], parameters => [ {code, hoconsc:mk(hoconsc:enum(emqx_dashboard_error_code:all()), #{ diff --git a/apps/emqx_exhook/src/emqx_exhook_api.erl b/apps/emqx_exhook/src/emqx_exhook_api.erl index 62f616903..921a4df68 100644 --- a/apps/emqx_exhook/src/emqx_exhook_api.erl +++ b/apps/emqx_exhook/src/emqx_exhook_api.erl @@ -41,7 +41,7 @@ -import(hoconsc, [mk/1, mk/2, ref/1, enum/1, array/1, map/2]). -import(emqx_dashboard_swagger, [schema_with_example/2, error_codes/2]). --define(TAGS, [<<"exhooks">>]). +-define(TAGS, [<<"ExHook">>]). -define(NOT_FOURD, 'NOT_FOUND'). -define(BAD_REQUEST, 'BAD_REQUEST'). -define(BAD_RPC, 'BAD_RPC'). diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl index e915e0fa2..bb622894b 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl @@ -182,7 +182,7 @@ schema("/gateways/:name/authentication") -> 'operationId' => authn, get => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(get_authn), summary => <<"Get Authenticator Configuration">>, parameters => params_gateway_name_in_path(), @@ -196,7 +196,7 @@ schema("/gateways/:name/authentication") -> }, put => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(update_authn), summary => <<"Update Authenticator Configuration">>, parameters => params_gateway_name_in_path(), @@ -206,7 +206,7 @@ schema("/gateways/:name/authentication") -> }, post => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(add_authn), summary => <<"Create an Authenticator for a Gateway">>, parameters => params_gateway_name_in_path(), @@ -216,7 +216,7 @@ schema("/gateways/:name/authentication") -> }, delete => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(delete_authn), summary => <<"Delete the Gateway Authenticator">>, parameters => params_gateway_name_in_path(), @@ -229,7 +229,7 @@ schema("/gateways/:name/authentication/users") -> 'operationId' => users, get => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(list_users), summary => <<"List users for a Gateway Authenticator">>, parameters => params_gateway_name_in_path() ++ @@ -247,7 +247,7 @@ schema("/gateways/:name/authentication/users") -> }, post => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(add_user), summary => <<"Add User for a Gateway Authenticator">>, parameters => params_gateway_name_in_path(), @@ -271,7 +271,7 @@ schema("/gateways/:name/authentication/users/:uid") -> 'operationId' => users_insta, get => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(get_user), summary => <<"Get User Info for a Gateway Authenticator">>, parameters => params_gateway_name_in_path() ++ @@ -288,7 +288,7 @@ schema("/gateways/:name/authentication/users/:uid") -> }, put => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(update_user), summary => <<"Update User Info for a Gateway Authenticator">>, parameters => params_gateway_name_in_path() ++ @@ -309,7 +309,7 @@ schema("/gateways/:name/authentication/users/:uid") -> }, delete => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(delete_user), summary => <<"Delete User for a Gateway Authenticator">>, parameters => params_gateway_name_in_path() ++ diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index 64ec46105..ab82a4716 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -467,7 +467,7 @@ schema("/gateways/:name/clients") -> 'operationId' => clients, get => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(list_clients), summary => <<"List Gateway's Clients">>, parameters => params_client_query(), @@ -480,7 +480,7 @@ schema("/gateways/:name/clients/:clientid") -> 'operationId' => clients_insta, get => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(get_client), summary => <<"Get Client Info">>, parameters => params_client_insta(), @@ -489,7 +489,7 @@ schema("/gateways/:name/clients/:clientid") -> }, delete => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(kick_client), summary => <<"Kick out Client">>, parameters => params_client_insta(), @@ -502,7 +502,7 @@ schema("/gateways/:name/clients/:clientid/subscriptions") -> 'operationId' => subscriptions, get => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(list_subscriptions), summary => <<"List Client's Subscription">>, parameters => params_client_insta(), @@ -518,7 +518,7 @@ schema("/gateways/:name/clients/:clientid/subscriptions") -> }, post => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(add_subscription), summary => <<"Add Subscription for Client">>, parameters => params_client_insta(), @@ -542,7 +542,7 @@ schema("/gateways/:name/clients/:clientid/subscriptions/:topic") -> 'operationId' => subscriptions, delete => #{ - tag => [<<"Gateways">>], + tags => [<<"Gateways">>], desc => ?DESC(delete_subscription), summary => <<"Delete Client's Subscription">>, parameters => params_topic_name_in_path() ++ params_client_insta(), diff --git a/apps/emqx_management/src/emqx_mgmt_api_app.erl b/apps/emqx_management/src/emqx_mgmt_api_app.erl index 89311a8d1..7050ea1af 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_app.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_app.erl @@ -22,7 +22,7 @@ -export([api_spec/0, fields/1, paths/0, schema/1, namespace/0]). -export([api_key/2, api_key_by_name/2]). -export([validate_name/1]). --define(TAGS, [<<"API keys">>]). +-define(TAGS, [<<"API Keys">>]). namespace() -> "api_key". diff --git a/apps/emqx_management/src/emqx_mgmt_api_sys.erl b/apps/emqx_management/src/emqx_mgmt_api_sys.erl index 43fd9ee14..c7aeb1d95 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_sys.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_sys.erl @@ -31,7 +31,7 @@ -export([sys/2]). --define(TAGS, [<<"System topics">>]). +-define(TAGS, [<<"System Topics">>]). namespace() -> "sys". diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl index cd2f76f11..c24d043e6 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl @@ -25,7 +25,7 @@ -export([api_spec/0, paths/0, schema/1, fields/1, namespace/0]). -export([slow_subs/2, get_history/0, settings/2]). --define(TAGS, [<<"Slow subscriptions">>]). +-define(TAGS, [<<"Slow Subscriptions">>]). -import(hoconsc, [mk/2, ref/1, ref/2]). -import(emqx_mgmt_util, [bad_request/0]). From 93d3fec73fa1e3b5d6054289f44c12632addff8d Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 30 Sep 2022 23:14:19 +0800 Subject: [PATCH 029/165] chore(i18n): correct tags name --- apps/emqx_authn/src/emqx_authn_user_import_api.erl | 2 +- .../src/emqx_gateway_api_authn_user_import.erl | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_user_import_api.erl b/apps/emqx_authn/src/emqx_authn_user_import_api.erl index 47d4de81c..dbd4c3656 100644 --- a/apps/emqx_authn/src/emqx_authn_user_import_api.erl +++ b/apps/emqx_authn/src/emqx_authn_user_import_api.erl @@ -30,7 +30,7 @@ % Swagger -define(API_TAGS_GLOBAL, [<<"Authentication">>]). --define(API_TAGS_SINGLE, [<<"Listener authentication">>]). +-define(API_TAGS_SINGLE, [<<"Listener Authentication">>]). -export([ api_spec/0, diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl index 38036f7c7..0d90b4dc5 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl @@ -122,7 +122,9 @@ schema("/gateways/:name/authentication/import_users") -> 'operationId' => import_users, post => #{ + tags => [<<"Gateways">>], desc => ?DESC(emqx_gateway_api_authn, import_users), + summary => <<"Import Users">>, parameters => params_gateway_name_in_path(), 'requestBody' => emqx_dashboard_swagger:file_schema(filename), responses => @@ -134,7 +136,9 @@ schema("/gateways/:name/listeners/:id/authentication/import_users") -> 'operationId' => import_listener_users, post => #{ + tags => [<<"Gateways">>], desc => ?DESC(emqx_gateway_api_listeners, import_users), + summary => <<"Import Users">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), 'requestBody' => emqx_dashboard_swagger:file_schema(filename), @@ -144,6 +148,7 @@ schema("/gateways/:name/listeners/:id/authentication/import_users") -> }; schema(Path) -> emqx_gateway_utils:make_compatible_schema(Path, fun schema/1). + %%-------------------------------------------------------------------- %% params defines %%-------------------------------------------------------------------- @@ -155,7 +160,7 @@ params_gateway_name_in_path() -> binary(), #{ in => path, - desc => ?DESC(emqx_gateway_api, gateway_name), + desc => ?DESC(emqx_gateway_api, gateway_name_in_qs), example => <<"stomp">> } )} From 7d7b05ae4ea4bb2ee21556bcb8346c81c471af75 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 30 Sep 2022 23:32:23 +0800 Subject: [PATCH 030/165] chore: update app.src --- apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_management/src/emqx_management.app.src | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src index 937f201bb..1c5627a1f 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auto_subscribe, [ {description, "An OTP application"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {mod, {emqx_auto_subscribe_app, []}}, {applications, [ diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index c4a43d846..3f844f61d 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.4"}, + {vsn, "5.0.5"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index d9f09f00e..5f8b30bf4 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.5"}, + {vsn, "5.0.6"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]}, From d43b680f142d4b496e80c4f8ad056c4863b43ae7 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 1 Oct 2022 09:40:27 +0800 Subject: [PATCH 031/165] chore: add comments to explain`/status` cant be found in swagger docs --- apps/emqx_management/src/emqx_mgmt_api_status.erl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/emqx_management/src/emqx_mgmt_api_status.erl b/apps/emqx_management/src/emqx_mgmt_api_status.erl index e38ccfd69..68bad03e4 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_status.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_status.erl @@ -20,6 +20,13 @@ path/0 ]). +%% Note: Because swagger now requires an HTTP prefix (e.g. /api/v5), +%% but the `/status` does not require this fixed prefix. +%% +%% Changing the swagger framework was too big, so we implemented the `/status` +%% in a simple way first +%% +%% XXX: So the HTTP API docs generated by swagger can not find this API now path() -> "/status". From bc71149d79edc677494f6cb3f1a52be6b0932951 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 3 Oct 2022 17:37:09 +0200 Subject: [PATCH 032/165] chore: respect PKG_VSN in pkg-vsn.sh --- pkg-vsn.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg-vsn.sh b/pkg-vsn.sh index 7f50f03e2..281160de8 100755 --- a/pkg-vsn.sh +++ b/pkg-vsn.sh @@ -73,6 +73,12 @@ while [ "$#" -gt 0 ]; do esac done +# return immediately if version is already set +if [[ "${PKG_VSN:-novalue}" != novalue && "${LONG_VERSION:-novalue}" != 'yes' ]]; then + echo "$PKG_VSN" + exit 0 +fi + case "${PROFILE}" in *enterprise*) RELEASE_EDITION="EMQX_RELEASE_EE" From b3a7fd26dc7699192dd9ee9818d08aa1efa28e17 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 5 Oct 2022 10:25:58 +0200 Subject: [PATCH 033/165] chore: stat to resolve script dir on macos and realpath otherwise --- bin/emqx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index 36762f1b1..811bb2981 100755 --- a/bin/emqx +++ b/bin/emqx @@ -7,7 +7,11 @@ set -euo pipefail DEBUG="${DEBUG:-0}" [ "$DEBUG" -eq 1 ] && set -x -RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" +if [ "$(uname -s)" == 'Darwin' ]; then + RUNNER_ROOT_DIR="$(cd "$(dirname "$(stat -f%R "$0" || echo "$0")")"/..; pwd -P)" +else + RUNNER_ROOT_DIR="$(cd "$(dirname "$(realpath "$0" || echo "$0")")"/..; pwd -P)" +fi # shellcheck disable=SC1090,SC1091 . "$RUNNER_ROOT_DIR"/releases/emqx_vars From abb941d161df10182ca9ab90a9e4f7a72228c70b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 5 Oct 2022 17:08:36 +0200 Subject: [PATCH 034/165] docs: remove port 8081 port 8081 is the management API in 4.x in 5.0 api port is 18083 (same as for serving the dashboard) --- Dockerfile.ubuntu20.04.runner | 5 ++--- README-CN.md | 6 ------ README-RU.md | 8 +------- README.md | 6 ------ apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl | 8 -------- deploy/charts/emqx-enterprise/README.md | 3 +-- deploy/charts/emqx-enterprise/values.yaml | 7 ++----- deploy/charts/emqx/README.md | 3 +-- deploy/charts/emqx/values.yaml | 7 ++----- deploy/docker/Dockerfile | 5 ++--- deploy/docker/Dockerfile.alpine | 5 ++--- 11 files changed, 13 insertions(+), 50 deletions(-) diff --git a/Dockerfile.ubuntu20.04.runner b/Dockerfile.ubuntu20.04.runner index 124021c89..1bb44a6e9 100644 --- a/Dockerfile.ubuntu20.04.runner +++ b/Dockerfile.ubuntu20.04.runner @@ -27,15 +27,14 @@ VOLUME ["/opt/emqx/log", "/opt/emqx/data"] # emqx will occupy these port: # - 1883 port for MQTT -# - 8081 for mgmt API # - 8083 for WebSocket/HTTP # - 8084 for WSS/HTTPS # - 8883 port for MQTT(SSL) # - 11883 port for internal MQTT/TCP -# - 18083 for dashboard +# - 18083 for dashboard and API # - 4370 default Erlang distrbution port # - 5369 for backplain gen_rpc -EXPOSE 1883 8081 8083 8084 8883 11883 18083 4370 5369 +EXPOSE 1883 8083 8084 8883 11883 18083 4370 5369 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] diff --git a/README-CN.md b/README-CN.md index 250d3e1c6..ea584c66a 100644 --- a/README-CN.md +++ b/README-CN.md @@ -32,12 +32,6 @@ EMQX 自 2013 年在 GitHub 发布开源版本以来,获得了来自 50 多个 docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx:latest ``` -或直接试用 EMQX 企业版(已内置 10 个并发连接的永不过期 License) - -``` -docker run -d --name emqx-ee -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx-ee:latest -``` - 接下来请参考 [入门指南](https://www.emqx.io/docs/zh/v5.0/getting-started/getting-started.html#启动-emqx) 开启您的 EMQX 之旅。 #### 在 Kubernetes 上运行 EMQX 集群 diff --git a/README-RU.md b/README-RU.md index cd8943795..76b1f0ae0 100644 --- a/README-RU.md +++ b/README-RU.md @@ -28,13 +28,7 @@ #### Установка EMQX с помощью Docker ``` -docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx -``` - -Или запустите EMQX Enterprise со встроенной бессрочной лицензией на 10 соединений. - -``` -docker run -d --name emqx-ee -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx-ee:latest +docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx ``` Чтобы ознакомиться с функциональностью EMQX, пожалуйста, следуйте [руководству по началу работы](https://www.emqx.io/docs/en/v5.0/getting-started/getting-started.html#start-emqx). diff --git a/README.md b/README.md index 4461f70c6..9543cf0de 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,6 @@ The simplest way to set up EMQX is to create a managed deployment with EMQX Clou docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx:latest ``` -Or install EMQX Enterprise with a built-in license for ten connections that never expire. - -``` -docker run -d --name emqx-ee -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx-ee:latest -``` - Next, please follow the [getting started guide](https://www.emqx.io/docs/en/v5.0/getting-started/getting-started.html#start-emqx) to tour the EMQX features. #### Run EMQX cluster on kubernetes diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index b3735109f..cb9c77657 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -74,14 +74,6 @@ end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_management]), mria:stop(). -set_special_configs(emqx_management) -> - Listeners = #{http => #{port => 8081}}, - Config = #{ - listeners => Listeners, - applications => [#{id => "admin", secret => "public"}] - }, - emqx_config:put([emqx_management], Config), - ok; set_special_configs(emqx_dashboard) -> emqx_dashboard_api_test_helpers:set_default_config(), ok; diff --git a/deploy/charts/emqx-enterprise/README.md b/deploy/charts/emqx-enterprise/README.md index 9c3762fdd..a579af70d 100644 --- a/deploy/charts/emqx-enterprise/README.md +++ b/deploy/charts/emqx-enterprise/README.md @@ -60,10 +60,9 @@ The following table lists the configurable parameters of the emqx chart and thei | `service.type` | Kubernetes Service type. | ClusterIP | | `service.mqtt` | Port for MQTT. | 1883 | | `service.mqttssl` | Port for MQTT(SSL). | 8883 | -| `service.mgmt` | Port for mgmt API. | 8081 | | `service.ws` | Port for WebSocket/HTTP. | 8083 | | `service.wss` | Port for WSS/HTTPS. | 8084 | -| `service.dashboard` | Port for dashboard. | 18083 | +| `service.dashboard` | Port for dashboard and API. | 18083 | | `service.nodePorts.mqtt` | Kubernetes node port for MQTT. | nil | | `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | nil | | `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | nil | diff --git a/deploy/charts/emqx-enterprise/values.yaml b/deploy/charts/emqx-enterprise/values.yaml index aa61a62ea..7827d6afb 100644 --- a/deploy/charts/emqx-enterprise/values.yaml +++ b/deploy/charts/emqx-enterprise/values.yaml @@ -123,19 +123,16 @@ service: ## Port for MQTT(SSL) ## mqttssl: 8883 - ## Port for mgmt API - ## - mgmt: 8081 ## Port for WebSocket/HTTP ## ws: 8083 ## Port for WSS/HTTPS ## wss: 8084 - ## Port for dashboard + ## Port for dashboard and API ## dashboard: 18083 - ## Port for dashboard HTTPS + ## Port for dashboard and API over HTTPS ## # dashboardtls: 18084 ## Specify the nodePort(s) value for the LoadBalancer and NodePort service types. diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index ed331619d..84b5ab4c9 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -60,10 +60,9 @@ The following table lists the configurable parameters of the emqx chart and thei | `service.type` | Kubernetes Service type. | ClusterIP | | `service.mqtt` | Port for MQTT. | 1883 | | `service.mqttssl` | Port for MQTT(SSL). | 8883 | -| `service.mgmt` | Port for mgmt API. | 8081 | | `service.ws` | Port for WebSocket/HTTP. | 8083 | | `service.wss` | Port for WSS/HTTPS. | 8084 | -| `service.dashboard` | Port for dashboard. | 18083 | +| `service.dashboard` | Port for dashboard and API. | 18083 | | `service.nodePorts.mqtt` | Kubernetes node port for MQTT. | nil | | `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | nil | | `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | nil | diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 5bf7377f4..b648f070f 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -125,19 +125,16 @@ service: ## Port for MQTT(SSL) ## mqttssl: 8883 - ## Port for mgmt API - ## - mgmt: 8081 ## Port for WebSocket/HTTP ## ws: 8083 ## Port for WSS/HTTPS ## wss: 8084 - ## Port for dashboard + ## Port for dashboard and API ## dashboard: 18083 - ## Port for dashboard HTTPS + ## Port for dashboard and API over HTTPS ## # dashboardtls: 18084 ## Specify the nodePort(s) value for the LoadBalancer and NodePort service types. diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 50e1187ab..51b4dbd0c 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -49,15 +49,14 @@ VOLUME ["/opt/emqx/log", "/opt/emqx/data"] # emqx will occupy these port: # - 1883 port for MQTT -# - 8081 for mgmt API # - 8083 for WebSocket/HTTP # - 8084 for WSS/HTTPS # - 8883 port for MQTT(SSL) # - 11883 port for internal MQTT/TCP -# - 18083 for dashboard +# - 18083 for dashboard and API # - 4370 default Erlang distribution port # - 5369 for backplain gen_rpc -EXPOSE 1883 8081 8083 8084 8883 11883 18083 4370 5369 +EXPOSE 1883 8083 8084 8883 11883 18083 4370 5369 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] diff --git a/deploy/docker/Dockerfile.alpine b/deploy/docker/Dockerfile.alpine index 6368a0b51..dc16afc54 100644 --- a/deploy/docker/Dockerfile.alpine +++ b/deploy/docker/Dockerfile.alpine @@ -62,15 +62,14 @@ VOLUME ["/opt/emqx/log", "/opt/emqx/data"] # emqx will occupy these port: # - 1883 port for MQTT -# - 8081 for mgmt API # - 8083 for WebSocket/HTTP # - 8084 for WSS/HTTPS # - 8883 port for MQTT(SSL) # - 11883 port for internal MQTT/TCP -# - 18083 for dashboard +# - 18083 for dashboard and API # - 4370 default Erlang distrbution port # - 5369 for backplain gen_rpc -EXPOSE 1883 8081 8083 8084 8883 11883 18083 4370 5369 +EXPOSE 1883 8083 8084 8883 11883 18083 4370 5369 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] From e8279a02ef2d931c7045b703e975bf4eaea1a1a2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 5 Oct 2022 21:27:42 +0200 Subject: [PATCH 035/165] fix(shared): re-dispatch inflight (QoS1) and mqueue messages when session terminates (not due to take over) shared delivery should be re-dispatched to other members in the group --- CHANGES-5.0.md | 1 + apps/emqx/src/emqx_mqueue.erl | 22 +- apps/emqx/src/emqx_session.erl | 40 ++-- apps/emqx/src/emqx_shared_sub.erl | 53 ++++- apps/emqx/test/emqx_shared_sub_SUITE.erl | 247 +++++++++++++++++------ 5 files changed, 270 insertions(+), 93 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 20d972096..af8aaecf1 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -10,6 +10,7 @@ * Fix GET /listeners API crash When some nodes still in initial configuration. [#9002](https://github.com/emqx/emqx/pull/9002) * Fix empty variable interpolation in authentication and authorization. Placeholders for undefined variables are rendered now as empty strings and do not cause errors anymore. [#8963](https://github.com/emqx/emqx/pull/8963) * Fix the latency statistics error of the slow subscription module when `stats_type` is `internal` or `response`. [#8986](https://github.com/emqx/emqx/pull/8986) +* Redispatch shared subscription messages. # 5.0.8 diff --git a/apps/emqx/src/emqx_mqueue.erl b/apps/emqx/src/emqx_mqueue.erl index f8556bc39..ae55d4b6e 100644 --- a/apps/emqx/src/emqx_mqueue.erl +++ b/apps/emqx/src/emqx_mqueue.erl @@ -66,7 +66,8 @@ in/2, out/1, stats/1, - dropped/1 + dropped/1, + to_list/1 ]). -define(NO_PRIORITY_TABLE, disabled). @@ -109,7 +110,7 @@ dropped = 0 :: count(), p_table = ?NO_PRIORITY_TABLE :: p_table(), default_p = ?LOWEST_PRIORITY :: priority(), - q = ?PQUEUE:new() :: pq(), + q = emqx_pqueue:new() :: pq(), shift_opts :: #shift_opts{}, last_prio :: non_neg_integer() | undefined, p_credit :: non_neg_integer() | undefined @@ -118,7 +119,7 @@ -type mqueue() :: #mqueue{}. -spec init(options()) -> mqueue(). -init(Opts = #{max_len := MaxLen0, store_qos0 := QoS_0}) -> +init(Opts = #{max_len := MaxLen0, store_qos0 := Qos0}) -> MaxLen = case (is_integer(MaxLen0) andalso MaxLen0 > ?MAX_LEN_INFINITY) of true -> MaxLen0; @@ -126,7 +127,7 @@ init(Opts = #{max_len := MaxLen0, store_qos0 := QoS_0}) -> end, #mqueue{ max_len = MaxLen, - store_qos0 = QoS_0, + store_qos0 = Qos0, p_table = get_opt(priorities, Opts, ?NO_PRIORITY_TABLE), default_p = get_priority_opt(Opts), shift_opts = get_shift_opt(Opts) @@ -152,6 +153,19 @@ len(#mqueue{len = Len}) -> Len. max_len(#mqueue{max_len = MaxLen}) -> MaxLen. +%% @doc Return all queued items in a list. +-spec to_list(mqueue()) -> list(). +to_list(MQ) -> + to_list(MQ, []). + +to_list(MQ, Acc) -> + case out(MQ) of + {empty, _MQ} -> + lists:reverse(Acc); + {{value, Msg}, Q1} -> + to_list(Q1, [Msg | Acc]) + end. + %% @doc Return number of dropped messages. -spec dropped(mqueue()) -> count(). dropped(#mqueue{dropped = Dropped}) -> Dropped. diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index 8ce8a1802..2e79bcfb1 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -801,7 +801,8 @@ replay(ClientInfo, Session = #session{inflight = Inflight}) -> -spec terminate(emqx_types:clientinfo(), Reason :: term(), session()) -> ok. terminate(ClientInfo, Reason, Session) -> run_terminate_hooks(ClientInfo, Reason, Session), - redispatch_shared_messages(Session), + Reason =/= takenover andalso + redispatch_shared_messages(Session), ok. run_terminate_hooks(ClientInfo, discarded, Session) -> @@ -811,29 +812,20 @@ run_terminate_hooks(ClientInfo, takenover, Session) -> run_terminate_hooks(ClientInfo, Reason, Session) -> run_hook('session.terminated', [ClientInfo, Reason, info(Session)]). -redispatch_shared_messages(#session{inflight = Inflight}) -> - InflightList = emqx_inflight:to_list(Inflight), - lists:foreach( - fun - %% Only QoS1 messages get redispatched, because QoS2 messages - %% must be sent to the same client, once they're in flight - ({_, #inflight_data{message = #message{qos = ?QOS_2} = Msg}}) -> - ?SLOG(warning, #{msg => qos2_lost_no_redispatch}, #{message => Msg}); - ({_, #inflight_data{message = #message{topic = Topic, qos = ?QOS_1} = Msg}}) -> - case emqx_shared_sub:get_group(Msg) of - {ok, Group} -> - %% Note that dispatch is called with self() in failed subs - %% This is done to avoid dispatching back to caller - Delivery = #delivery{sender = self(), message = Msg}, - emqx_shared_sub:dispatch(Group, Topic, Delivery, [self()]); - _ -> - false - end; - (_) -> - ok - end, - InflightList - ). +redispatch_shared_messages(#session{inflight = Inflight, mqueue = Q}) -> + AllInflights = emqx_inflight:to_list(fun sort_fun/2, Inflight), + F = fun + ({_PacketId, #inflight_data{message = #message{qos = ?QOS_1} = Msg}}) -> + %% For QoS 2, here is what the spec says: + %% If the Client's Session terminates before the Client reconnects, + %% the Server MUST NOT send the Application Message to any other + %% subscribed Client [MQTT-4.8.2-5]. + {true, Msg}; + ({_PacketId, #inflight_data{}}) -> + false + end, + InflightList = lists:filtermap(F, AllInflights), + emqx_shared_sub:redispatch(InflightList ++ emqx_mqueue:to_list(Q)). -compile({inline, [run_hook/2]}). run_hook(Name, Args) -> diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 0527cbfe7..0f7e082eb 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -39,7 +39,8 @@ -export([ dispatch/3, dispatch/4, - do_dispatch_with_ack/4 + do_dispatch_with_ack/4, + redispatch/1 ]). -export([ @@ -96,6 +97,9 @@ -define(ACK, shared_sub_ack). -define(NACK(Reason), {shared_sub_nack, Reason}). -define(NO_ACK, no_ack). +-define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}). + +-type redispatch_to() :: ?REDISPATCH_TO(emqx_topic:group(), emqx_topic:topic()). -record(state, {pmon}). @@ -144,7 +148,8 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) -> false -> {error, no_subscribers}; {Type, SubPid} -> - case do_dispatch(SubPid, Group, Topic, Msg, Type) of + Msg1 = with_redispatch_to(Msg, Group, Topic), + case do_dispatch(SubPid, Group, Topic, Msg1, Type) of ok -> {ok, 1}; {error, _Reason} -> @@ -223,6 +228,50 @@ without_group_ack(Msg) -> get_group_ack(Msg) -> emqx_message:get_header(shared_dispatch_ack, Msg, ?NO_ACK). +with_redispatch_to(#message{qos = ?QOS_0} = Msg, _Group, _Topic) -> + Msg; +with_redispatch_to(Msg, Group, Topic) -> + emqx_message:set_headers(#{redispatch_to => ?REDISPATCH_TO(Group, Topic)}, Msg). + +%% @hidden Redispatch is neede only for the messages with redispatch_to header added. +is_redispatch_needed(#message{} = Msg) -> + case get_redispatch_to(Msg) of + ?REDISPATCH_TO(_, _) -> + true; + _ -> + false + end. + +%% @doc Redispatch shared deliveries to other members in the group. +redispatch(Messages0) -> + Messages = lists:filter(fun is_redispatch_needed/1, Messages0), + case length(Messages) of + L when L > 0 -> + ?SLOG(info, #{ + msg => "redispatching_shared_subscription_message", + count => L + }), + lists:foreach(fun redispatch_shared_message/1, Messages); + _ -> + ok + end. + +redispatch_shared_message(#message{} = Msg) -> + %% As long as it's still a #message{} record in inflight, + %% we should try to re-dispatch + ?REDISPATCH_TO(Group, Topic) = get_redispatch_to(Msg), + %% Note that dispatch is called with self() in failed subs + %% This is done to avoid dispatching back to caller + Delivery = #delivery{sender = self(), message = Msg}, + dispatch(Group, Topic, Delivery, [self()]). + +%% @hidden Return the `redispatch_to` group-topic in the message header. +%% `false` is returned if the message is not a shared dispatch. +%% or when it's a QoS 0 message. +-spec get_redispatch_to(emqx_types:message()) -> redispatch_to() | false. +get_redispatch_to(Msg) -> + emqx_message:get_header(redispatch_to, Msg, false). + -spec is_ack_required(emqx_types:message()) -> boolean(). is_ack_required(Msg) -> ?NO_ACK =/= get_group_ack(Msg). diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 8616028ca..5089a3a24 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -25,10 +25,20 @@ -define(SUITE, ?MODULE). --define(wait(For, Timeout), - emqx_common_test_helpers:wait_for( - ?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout - ) +-define(WAIT(TIMEOUT, PATTERN, Res), + (fun() -> + receive + PATTERN -> + Res; + Other -> + ct:fail(#{ + expected => ??PATTERN, + got => Other + }) + after TIMEOUT -> + ct:fail({timeout, ??PATTERN}) + end + end)() ). -define(ack, shared_sub_ack). @@ -45,10 +55,26 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([]). -t_is_ack_required(_) -> +init_per_testcase(Case, Config) -> + try + ?MODULE:Case({'init', Config}) + catch + error:function_clause -> + Config + end. + +end_per_testcase(Case, Config) -> + try + ?MODULE:Case({'end', Config}) + catch + error:function_clause -> + ok + end. + +t_is_ack_required(Config) when is_list(Config) -> ?assertEqual(false, emqx_shared_sub:is_ack_required(#message{headers = #{}})). -t_maybe_nack_dropped(_) -> +t_maybe_nack_dropped(Config) when is_list(Config) -> ?assertEqual(false, emqx_shared_sub:maybe_nack_dropped(#message{headers = #{}})), Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}}, ?assertEqual(true, emqx_shared_sub:maybe_nack_dropped(Msg)), @@ -60,7 +86,7 @@ t_maybe_nack_dropped(_) -> end ). -t_nack_no_connection(_) -> +t_nack_no_connection(Config) when is_list(Config) -> Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}}, ?assertEqual(ok, emqx_shared_sub:nack_no_connection(Msg)), ?assertEqual( @@ -71,7 +97,7 @@ t_nack_no_connection(_) -> end ). -t_maybe_ack(_) -> +t_maybe_ack(Config) when is_list(Config) -> ?assertEqual(#message{headers = #{}}, emqx_shared_sub:maybe_ack(#message{headers = #{}})), Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}}, ?assertEqual( @@ -86,10 +112,7 @@ t_maybe_ack(_) -> end ). -% t_subscribers(_) -> -% error('TODO'). - -t_random_basic(_) -> +t_random_basic(Config) when is_list(Config) -> ok = ensure_config(random), ClientId = <<"ClientId">>, Topic = <<"foo">>, @@ -121,7 +144,7 @@ t_random_basic(_) -> %% After the connection for the 2nd session is also closed, %% i.e. when all clients are offline, the following message(s) %% should be delivered randomly. -t_no_connection_nack(_) -> +t_no_connection_nack(Config) when is_list(Config) -> ok = ensure_config(sticky), Publisher = <<"publisher">>, Subscriber1 = <<"Subscriber1">>, @@ -153,54 +176,22 @@ t_no_connection_nack(_) -> %% This is the connection which was picked by broker to dispatch (sticky) for 1st message ?assertMatch([#{packet_id := 1}], recv_msgs(1)), - %% Now kill the connection, expect all following messages to be delivered to the other - %% subscriber. - %emqx_mock_client:stop(ConnPid), - %% sleep then make synced calls to session processes to ensure that - %% the connection pid's 'EXIT' message is propagated to the session process - %% also to be sure sessions are still alive - % timer:sleep(2), - % _ = emqx_session:info(SPid1), - % _ = emqx_session:info(SPid2), - % %% Now we know what is the other still alive connection - % [TheOtherConnPid] = [SubConnPid1, SubConnPid2] -- [ConnPid], - % %% Send some more messages - % PacketIdList = lists:seq(2, 10), - % lists:foreach(fun(Id) -> - % SendF(Id), - % ?wait(Received(Id, TheOtherConnPid), 1000) - % end, PacketIdList), - % %% Now close the 2nd (last connection) - % emqx_mock_client:stop(TheOtherConnPid), - % timer:sleep(2), - % %% both sessions should have conn_pid = undefined - % ?assertEqual({conn_pid, undefined}, lists:keyfind(conn_pid, 1, emqx_session:info(SPid1))), - % ?assertEqual({conn_pid, undefined}, lists:keyfind(conn_pid, 1, emqx_session:info(SPid2))), - % %% send more messages, but all should be queued in session state - % lists:foreach(fun(Id) -> SendF(Id) end, PacketIdList), - % {_, L1} = lists:keyfind(mqueue_len, 1, emqx_session:info(SPid1)), - % {_, L2} = lists:keyfind(mqueue_len, 1, emqx_session:info(SPid2)), - % ?assertEqual(length(PacketIdList), L1 + L2), - % %% clean up - % emqx_mock_client:close_session(PubConnPid), - % emqx_sm:close_session(SPid1), - % emqx_sm:close_session(SPid2), ok. -t_random(_) -> +t_random(Config) when is_list(Config) -> ok = ensure_config(random, true), test_two_messages(random). -t_round_robin(_) -> +t_round_robin(Config) when is_list(Config) -> ok = ensure_config(round_robin, true), test_two_messages(round_robin). -t_round_robin_per_group(_) -> +t_round_robin_per_group(Config) when is_list(Config) -> ok = ensure_config(round_robin_per_group, true), test_two_messages(round_robin_per_group). %% this would fail if executed with the standard round_robin strategy -t_round_robin_per_group_even_distribution_one_group(_) -> +t_round_robin_per_group_even_distribution_one_group(Config) when is_list(Config) -> ok = ensure_config(round_robin_per_group, true), Topic = <<"foo/bar">>, Group = <<"group1">>, @@ -264,7 +255,7 @@ t_round_robin_per_group_even_distribution_one_group(_) -> ), ok. -t_round_robin_per_group_even_distribution_two_groups(_) -> +t_round_robin_per_group_even_distribution_two_groups(Config) when is_list(Config) -> ok = ensure_config(round_robin_per_group, true), Topic = <<"foo/bar">>, {ok, ConnPid1} = emqtt:start_link([{clientid, <<"C0">>}]), @@ -350,19 +341,19 @@ t_round_robin_per_group_even_distribution_two_groups(_) -> ), ok. -t_sticky(_) -> +t_sticky(Config) when is_list(Config) -> ok = ensure_config(sticky, true), test_two_messages(sticky). -t_hash(_) -> +t_hash(Config) when is_list(Config) -> ok = ensure_config(hash, false), test_two_messages(hash). -t_hash_clinetid(_) -> +t_hash_clinetid(Config) when is_list(Config) -> ok = ensure_config(hash_clientid, false), test_two_messages(hash_clientid). -t_hash_topic(_) -> +t_hash_topic(Config) when is_list(Config) -> ok = ensure_config(hash_topic, false), ClientId1 = <<"ClientId1">>, ClientId2 = <<"ClientId2">>, @@ -407,7 +398,7 @@ t_hash_topic(_) -> ok. %% if the original subscriber dies, change to another one alive -t_not_so_sticky(_) -> +t_not_so_sticky(Config) when is_list(Config) -> ok = ensure_config(sticky), ClientId1 = <<"ClientId1">>, ClientId2 = <<"ClientId2">>, @@ -481,7 +472,7 @@ last_message(ExpectedPayload, Pids, Timeout) -> <<"not yet?">> end. -t_dispatch(_) -> +t_dispatch(Config) when is_list(Config) -> ok = ensure_config(random), Topic = <<"foo">>, ?assertEqual( @@ -494,13 +485,13 @@ t_dispatch(_) -> emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}}) ). -t_uncovered_func(_) -> +t_uncovered_func(Config) when is_list(Config) -> ignored = gen_server:call(emqx_shared_sub, ignored), ok = gen_server:cast(emqx_shared_sub, ignored), ignored = emqx_shared_sub ! ignored, {mnesia_table_event, []} = emqx_shared_sub ! {mnesia_table_event, []}. -t_per_group_config(_) -> +t_per_group_config(Config) when is_list(Config) -> ok = ensure_group_config(#{ <<"local_group">> => local, <<"round_robin_group">> => round_robin, @@ -521,7 +512,7 @@ t_per_group_config(_) -> test_two_messages(round_robin_per_group, <<"round_robin_per_group_group">>), test_two_messages(round_robin_per_group, <<"round_robin_per_group_group">>). -t_local(_) -> +t_local(Config) when is_list(Config) -> GroupConfig = #{ <<"local_group">> => local, <<"round_robin_group">> => round_robin, @@ -567,7 +558,7 @@ t_local(_) -> ?assertNotEqual(UsedSubPid1, UsedSubPid2), ok. -t_remote(_) -> +t_remote(Config) when is_list(Config) -> %% This testcase verifies dispatching of shared messages to the remote nodes via backplane API. %% %% In this testcase we start two EMQX nodes: local and remote. @@ -620,7 +611,7 @@ t_remote(_) -> stop_slave(Node) end. -t_local_fallback(_) -> +t_local_fallback(Config) when is_list(Config) -> ok = ensure_group_config(#{ <<"local_group">> => local, <<"round_robin_group">> => round_robin, @@ -653,9 +644,14 @@ t_local_fallback(_) -> %% This one tests that broker tries to select another shared subscriber %% If the first one doesn't return an ACK -t_redispatch(_) -> - ok = ensure_config(sticky, true), +t_redispatch_qos1_with_ack(Config) when is_list(Config) -> + test_redispatch_qos1(Config, true). +t_redispatch_qos1_no_ack(Config) when is_list(Config) -> + test_redispatch_qos1(Config, false). + +test_redispatch_qos1(_Config, AckEnabled) -> + ok = ensure_config(sticky, AckEnabled), Group = <<"group1">>, Topic = <<"foo/bar">>, ClientId1 = <<"ClientId1">>, @@ -682,10 +678,135 @@ t_redispatch(_) -> emqtt:stop(UsedSubPid2), ok. +%% No ack, QoS 2 subscriptions, +%% client1 receives one message, send pubrec, then suspend +%% client2 acts normal (auto_ack=true) +%% Expected behaviour: +%% the messages sent to client1's inflight and mq are re-dispatched after client1 is down +t_dispatch_qos2({init, Config}) when is_list(Config) -> + emqx_config:put_zone_conf(default, [mqtt, max_inflight], 1), + Config; +t_dispatch_qos2({'end', Config}) when is_list(Config) -> + emqx_config:put_zone_conf(default, [mqtt, max_inflight], 0); +t_dispatch_qos2(Config) when is_list(Config) -> + ok = ensure_config(round_robin, _AckEnabled = false), + Topic = <<"foo/bar/1">>, + ClientId1 = <<"ClientId1">>, + ClientId2 = <<"ClientId2">>, + + {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}, {auto_ack, false}]), + {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}, {auto_ack, true}]), + {ok, _} = emqtt:connect(ConnPid1), + {ok, _} = emqtt:connect(ConnPid2), + + emqtt:subscribe(ConnPid1, {<<"$share/group/foo/bar/#">>, 2}), + emqtt:subscribe(ConnPid2, {<<"$share/group/foo/bar/#">>, 2}), + + Message1 = emqx_message:make(ClientId1, 2, Topic, <<"hello1">>), + Message2 = emqx_message:make(ClientId1, 2, Topic, <<"hello2">>), + Message3 = emqx_message:make(ClientId1, 2, Topic, <<"hello3">>), + Message4 = emqx_message:make(ClientId1, 2, Topic, <<"hello4">>), + ct:sleep(100), + + ok = sys:suspend(ConnPid1), + + %% One message is inflight + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message1)), + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message2)), + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message3)), + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message4)), + + MsgRec1 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P1}}, P1), + MsgRec2 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P2}}, P2), + %% assert hello2 > hello1 or hello4 > hello3 + ?assert(MsgRec2 > MsgRec1), + + sys:resume(ConnPid1), + %% emqtt subscriber automatically sends PUBREC, but since auto_ack is set to false + %% so it will never send PUBCOMP, hence EMQX should not attempt to send + %% the 4th message yet since max_inflight is 1. + MsgRec3 = ?WAIT(2000, {publish, #{client_pid := ConnPid1, payload := P3}}, P3), + ct:sleep(100), + %% no message expected + ?assertEqual([], collect_msgs(0)), + %% now kill client 1 + kill_process(ConnPid1), + %% client 2 should receive the message + MsgRec4 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P4}}, P4), + %% assert hello2 > hello1 or hello4 > hello3 + ?assert(MsgRec4 > MsgRec3), + emqtt:stop(ConnPid2), + ok. + +t_dispatch_qos0({init, Config}) when is_list(Config) -> + Config; +t_dispatch_qos0({'end', Config}) when is_list(Config) -> + ok; +t_dispatch_qos0(Config) when is_list(Config) -> + ok = ensure_config(round_robin, _AckEnabled = false), + Topic = <<"foo/bar/1">>, + ClientId1 = <<"ClientId1">>, + ClientId2 = <<"ClientId2">>, + + {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}, {auto_ack, false}]), + {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}, {auto_ack, true}]), + {ok, _} = emqtt:connect(ConnPid1), + {ok, _} = emqtt:connect(ConnPid2), + + %% subscribe with QoS 0 + emqtt:subscribe(ConnPid1, {<<"$share/group/foo/bar/#">>, 0}), + emqtt:subscribe(ConnPid2, {<<"$share/group/foo/bar/#">>, 0}), + + %% publish with QoS 2, but should be downgraded to 0 as the subscribers + %% subscribe with QoS 0 + Message1 = emqx_message:make(ClientId1, 2, Topic, <<"hello1">>), + Message2 = emqx_message:make(ClientId1, 2, Topic, <<"hello2">>), + Message3 = emqx_message:make(ClientId1, 2, Topic, <<"hello3">>), + Message4 = emqx_message:make(ClientId1, 2, Topic, <<"hello4">>), + ct:sleep(100), + + ok = sys:suspend(ConnPid1), + + ?assertMatch([_], emqx:publish(Message1)), + ?assertMatch([_], emqx:publish(Message2)), + ?assertMatch([_], emqx:publish(Message3)), + ?assertMatch([_], emqx:publish(Message4)), + + MsgRec1 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P1}}, P1), + MsgRec2 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P2}}, P2), + %% assert hello2 > hello1 or hello4 > hello3 + ?assert(MsgRec2 > MsgRec1), + + kill_process(ConnPid1), + %% expect no redispatch + ?assertEqual([], collect_msgs(timer:seconds(2))), + emqtt:stop(ConnPid2), + ok. + %%-------------------------------------------------------------------- %% help functions %%-------------------------------------------------------------------- +kill_process(Pid) -> + _ = unlink(Pid), + _ = monitor(process, Pid), + erlang:exit(Pid, kill), + receive + {'DOWN', _, process, Pid, _} -> + ok + end. + +collect_msgs(Timeout) -> + collect_msgs([], Timeout). + +collect_msgs(Acc, Timeout) -> + receive + Msg -> + collect_msgs([Msg | Acc], Timeout) + after Timeout -> + lists:reverse(Acc) + end. + ensure_config(Strategy) -> ensure_config(Strategy, _AckEnabled = true). From 1c29e2806a941a2240443511f01f2fe2c56a8e61 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 5 Oct 2022 22:23:07 +0200 Subject: [PATCH 036/165] test(shared): add a test case to ensure shared dispatch retry to ensure retry will not enter a dead loop --- CHANGES-5.0.md | 2 +- apps/emqx/src/emqx_channel.erl | 14 ++++++++-- apps/emqx/src/emqx_session.erl | 6 ++++- apps/emqx/src/emqx_shared_sub.erl | 10 +------ apps/emqx/test/emqx_shared_sub_SUITE.erl | 34 ++++++++++++++++++++++++ 5 files changed, 53 insertions(+), 13 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index af8aaecf1..1a08e41e8 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -10,7 +10,7 @@ * Fix GET /listeners API crash When some nodes still in initial configuration. [#9002](https://github.com/emqx/emqx/pull/9002) * Fix empty variable interpolation in authentication and authorization. Placeholders for undefined variables are rendered now as empty strings and do not cause errors anymore. [#8963](https://github.com/emqx/emqx/pull/8963) * Fix the latency statistics error of the slow subscription module when `stats_type` is `internal` or `response`. [#8986](https://github.com/emqx/emqx/pull/8986) -* Redispatch shared subscription messages. +* Redispatch shared subscription messages. [#9104](https://github.com/emqx/emqx/pull/9104) # 5.0.8 diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index d1a111dc5..742868694 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -997,8 +997,13 @@ maybe_nack(Delivers) -> lists:filter(fun not_nacked/1, Delivers). not_nacked({deliver, _Topic, Msg}) -> - not (emqx_shared_sub:is_ack_required(Msg) andalso - (ok == emqx_shared_sub:nack_no_connection(Msg))). + case emqx_shared_sub:is_ack_required(Msg) of + true -> + ok = emqx_shared_sub:nack_no_connection(Msg), + false; + false -> + true + end. maybe_mark_as_delivered(Session, Delivers) -> case emqx_session:info(is_persistent, Session) of @@ -1222,6 +1227,8 @@ handle_call( ChanInfo1 = info(NChannel), emqx_cm:set_chan_info(ClientId, ChanInfo1#{sockinfo => SockInfo}), reply(ok, reset_timer(alive_timer, NChannel)); +handle_call(get_mqueue, Channel) -> + reply({ok, get_mqueue(Channel)}, Channel); handle_call(Req, Channel) -> ?SLOG(error, #{msg => "unexpected_call", call => Req}), reply(ignored, Channel). @@ -2224,3 +2231,6 @@ get_mqtt_conf(Zone, Key, Default) -> set_field(Name, Value, Channel) -> Pos = emqx_misc:index_of(Name, record_info(fields, channel)), setelement(Pos + 1, Channel, Value). + +get_mqueue(#channel{session = Session}) -> + emqx_session:get_mqueue(Session). diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index 2e79bcfb1..b285d0a88 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -60,7 +60,8 @@ info/2, is_session/1, stats/1, - obtain_next_pkt_id/1 + obtain_next_pkt_id/1, + get_mqueue/1 ]). -export([ @@ -917,3 +918,6 @@ age(Now, Ts) -> Now - Ts. set_field(Name, Value, Session) -> Pos = emqx_misc:index_of(Name, record_info(fields, session)), setelement(Pos + 1, Session, Value). + +get_mqueue(#session{mqueue = Q}) -> + emqx_mqueue:to_list(Q). diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 0f7e082eb..975b403b9 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -47,8 +47,7 @@ maybe_ack/1, maybe_nack_dropped/1, nack_no_connection/1, - is_ack_required/1, - get_group/1 + is_ack_required/1 ]). %% for testing @@ -275,13 +274,6 @@ get_redispatch_to(Msg) -> -spec is_ack_required(emqx_types:message()) -> boolean(). is_ack_required(Msg) -> ?NO_ACK =/= get_group_ack(Msg). --spec get_group(emqx_types:message()) -> {ok, any()} | error. -get_group(Msg) -> - case get_group_ack(Msg) of - ?NO_ACK -> error; - {Group, _Sender, _Ref} -> {ok, Group} - end. - %% @doc Negative ack dropped message due to inflight window or message queue being full. -spec maybe_nack_dropped(emqx_types:message()) -> boolean(). maybe_nack_dropped(Msg) -> diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 5089a3a24..291286aa2 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -678,6 +678,40 @@ test_redispatch_qos1(_Config, AckEnabled) -> emqtt:stop(UsedSubPid2), ok. +t_qos1_random_dispatch_if_all_members_are_down(Config) when is_list(Config) -> + ok = ensure_config(sticky, true), + Group = <<"group1">>, + Topic = <<"foo/bar">>, + ClientId1 = <<"ClientId1">>, + ClientId2 = <<"ClientId2">>, + SubOpts = [{clean_start, false}], + {ok, ConnPub} = emqtt:start_link([{clientid, <<"pub">>}]), + {ok, _} = emqtt:connect(ConnPub), + + {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1} | SubOpts]), + {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2} | SubOpts]), + {ok, _} = emqtt:connect(ConnPid1), + {ok, _} = emqtt:connect(ConnPid2), + + emqtt:subscribe(ConnPid1, {<<"$share/", Group/binary, "/foo/bar">>, 1}), + emqtt:subscribe(ConnPid2, {<<"$share/", Group/binary, "/foo/bar">>, 1}), + + ok = emqtt:stop(ConnPid1), + ok = emqtt:stop(ConnPid2), + + [Pid1, Pid2] = emqx_shared_sub:subscribers(Group, Topic), + ?assert(is_process_alive(Pid1)), + ?assert(is_process_alive(Pid2)), + + {ok, _} = emqtt:publish(ConnPub, Topic, <<"hello11">>, 1), + ct:sleep(100), + {ok, Msgs1} = gen_server:call(Pid1, get_mqueue), + {ok, Msgs2} = gen_server:call(Pid2, get_mqueue), + %% assert the message is in mqueue (because socket is closed) + ?assertMatch([#message{payload = <<"hello11">>}], Msgs1 ++ Msgs2), + emqtt:stop(ConnPub), + ok. + %% No ack, QoS 2 subscriptions, %% client1 receives one message, send pubrec, then suspend %% client2 acts normal (auto_ack=true) From 5654778bd97f009054185f26f68cda07eda355a5 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 7 Oct 2022 09:31:11 +0200 Subject: [PATCH 037/165] build: check if quicer binary present before trying to sign it --- scripts/macos-sign-binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/macos-sign-binaries.sh b/scripts/macos-sign-binaries.sh index a69b9a49b..11b6b734d 100755 --- a/scripts/macos-sign-binaries.sh +++ b/scripts/macos-sign-binaries.sh @@ -52,4 +52,4 @@ codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=r codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup} codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/rocksdb-*/priv/liberocksdb.so codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so} -codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/quicer-*/priv/libquicer_nif.so +find "${REL_DIR}/lib/" -name libquicer_nif.so -exec codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime {} \; From 6d852b69a211eff04f8cfd87d709a7841f381953 Mon Sep 17 00:00:00 2001 From: firest Date: Sun, 9 Oct 2022 10:13:09 +0800 Subject: [PATCH 038/165] chore: update CHANGES --- CHANGES-5.0.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 04dffaf8e..d4f831284 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -1,14 +1,9 @@ -# 5.0.10 - -## Enhancements - -* Use milliseconds internally in emqx_delayed to store the publish time, improving precision.[#9060](https://github.com/emqx/emqx/pull/9060) - # 5.0.9 ## Enhancements * Add `cert_common_name` and `cert_subject` placeholder support for authz_http and authz_mongo.[#8973](https://github.com/emqx/emqx/pull/8973) +* Use milliseconds internally in emqx_delayed to store the publish time, improving precision.[#9060](https://github.com/emqx/emqx/pull/9060) ## Bug fixes From f0769cb765cbed054226123f983986481c42c8e1 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 10 Oct 2022 13:52:09 +0800 Subject: [PATCH 039/165] fix(shared_sub): kick session should not cause session message redispatch --- apps/emqx/src/emqx_session.erl | 9 ++- apps/emqx/test/emqx_shared_sub_SUITE.erl | 71 ++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index b285d0a88..6567bf5c8 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -802,8 +802,7 @@ replay(ClientInfo, Session = #session{inflight = Inflight}) -> -spec terminate(emqx_types:clientinfo(), Reason :: term(), session()) -> ok. terminate(ClientInfo, Reason, Session) -> run_terminate_hooks(ClientInfo, Reason, Session), - Reason =/= takenover andalso - redispatch_shared_messages(Session), + redispatch_shared_messages(Reason, Session), ok. run_terminate_hooks(ClientInfo, discarded, Session) -> @@ -813,7 +812,11 @@ run_terminate_hooks(ClientInfo, takenover, Session) -> run_terminate_hooks(ClientInfo, Reason, Session) -> run_hook('session.terminated', [ClientInfo, Reason, info(Session)]). -redispatch_shared_messages(#session{inflight = Inflight, mqueue = Q}) -> +redispatch_shared_messages(takenover, _Session) -> + ok; +redispatch_shared_messages(kicked, _Session) -> + ok; +redispatch_shared_messages(_Reason, #session{inflight = Inflight, mqueue = Q}) -> AllInflights = emqx_inflight:to_list(fun sort_fun/2, Inflight), F = fun ({_PacketId, #inflight_data{message = #message{qos = ?QOS_1} = Msg}}) -> diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 291286aa2..2cdf2bc05 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -817,6 +817,77 @@ t_dispatch_qos0(Config) when is_list(Config) -> emqtt:stop(ConnPid2), ok. +t_redispatch_when_kicked({init, Config}) when is_list(Config) -> + Config; +t_redispatch_when_kicked({'end', Config}) when is_list(Config) -> + ok; +t_redispatch_when_kicked(_) -> + ok = ensure_config(sticky, true), + + Group = <<"group1">>, + Topic = <<"foo/bar">>, + ClientId1 = <<"ClientId1">>, + ClientId2 = <<"ClientId2">>, + Parent = self(), + + %% emqx_cm:kick_session will cause the emqtt to exit abnormally + %% so here need to isolate emqtt with a separate process + MkSub = fun(ClientId) -> + {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}, {auto_ack, false}]), + {ok, _} = emqtt:connect(ConnPid), + {ok, _Props, _ReasonCodes} = emqtt:subscribe(ConnPid, { + <<"$share/", Group/binary, "/foo/bar">>, 1 + }), + + Loop = fun(Self) -> + receive + wait -> + case last_message(<<"hello1">>, [ConnPid], 6000) of + {true, _} -> + Parent ! {got, ClientId}; + _ -> + Parent ! nothing + end, + Self(Self); + stop -> + stop + end + end, + + Loop(Loop) + end, + + Receive = fun() -> + receive + {got, _} = Got -> Got; + nothing -> nothing + after 6000 -> + nothing + end + end, + + Subs = [erlang:spawn(fun() -> MkSub(ClientId) end) || ClientId <- [ClientId1, ClientId2]], + timer:sleep(500), + + Message = emqx_message:make(ClientId1, 1, Topic, <<"hello1">>), + emqx:publish(Message), + + [Sub ! wait || Sub <- Subs], + timer:sleep(200), + Res = Receive(), + ?assertMatch({got, _}, Res), + + {got, ClientId} = Res, + emqx_cm:kick_session(ClientId), + + [Sub ! wait || Sub <- Subs], + timer:sleep(200), + Res1 = Receive(), + ?assertMatch(nothing, Res1), + + [Sub ! stop || Sub <- Subs], + ok. + %%-------------------------------------------------------------------- %% help functions %%-------------------------------------------------------------------- From b4eb0f18f9b493fe953bdc03bd4c1dfe170da6d4 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 10 Oct 2022 18:06:55 +0800 Subject: [PATCH 040/165] fix(shared_subs): rename redispatch function name and simply test case --- apps/emqx/src/emqx_session.erl | 11 +++++++---- apps/emqx/test/emqx_shared_sub_SUITE.erl | 11 ++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index 6567bf5c8..4d1594a99 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -47,6 +47,7 @@ -include("emqx_mqtt.hrl"). -include("logger.hrl"). -include("types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -ifdef(TEST). -compile(export_all). @@ -802,7 +803,7 @@ replay(ClientInfo, Session = #session{inflight = Inflight}) -> -spec terminate(emqx_types:clientinfo(), Reason :: term(), session()) -> ok. terminate(ClientInfo, Reason, Session) -> run_terminate_hooks(ClientInfo, Reason, Session), - redispatch_shared_messages(Reason, Session), + maybe_redispatch_shared_messages(Reason, Session), ok. run_terminate_hooks(ClientInfo, discarded, Session) -> @@ -812,11 +813,13 @@ run_terminate_hooks(ClientInfo, takenover, Session) -> run_terminate_hooks(ClientInfo, Reason, Session) -> run_hook('session.terminated', [ClientInfo, Reason, info(Session)]). -redispatch_shared_messages(takenover, _Session) -> +maybe_redispatch_shared_messages(takenover, _Session) -> + ?tp(debug, ignore_redispatch_shared_messages, #{reason => takenover}), ok; -redispatch_shared_messages(kicked, _Session) -> +maybe_redispatch_shared_messages(kicked, _Session) -> + ?tp(debug, ignore_redispatch_shared_messages, #{reason => kicked}), ok; -redispatch_shared_messages(_Reason, #session{inflight = Inflight, mqueue = Q}) -> +maybe_redispatch_shared_messages(_Reason, #session{inflight = Inflight, mqueue = Q}) -> AllInflights = emqx_inflight:to_list(fun sort_fun/2, Inflight), F = fun ({_PacketId, #inflight_data{message = #message{qos = ?QOS_1} = Msg}}) -> diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 2cdf2bc05..f048df397 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -22,6 +22,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(SUITE, ?MODULE). @@ -869,23 +870,23 @@ t_redispatch_when_kicked(_) -> Subs = [erlang:spawn(fun() -> MkSub(ClientId) end) || ClientId <- [ClientId1, ClientId2]], timer:sleep(500), + ok = snabbkaffe:start_trace(), + Message = emqx_message:make(ClientId1, 1, Topic, <<"hello1">>), emqx:publish(Message), [Sub ! wait || Sub <- Subs], - timer:sleep(200), Res = Receive(), ?assertMatch({got, _}, Res), {got, ClientId} = Res, emqx_cm:kick_session(ClientId), - [Sub ! wait || Sub <- Subs], - timer:sleep(200), - Res1 = Receive(), - ?assertMatch(nothing, Res1), + Trace = snabbkaffe:collect_trace(500), + ?assertMatch([#{reason := kicked}], ?of_kind(ignore_redispatch_shared_messages, Trace)), [Sub ! stop || Sub <- Subs], + snabbkaffe:stop(), ok. %%-------------------------------------------------------------------- From 56000cbf3ecdbbd4ad19f3a8365c6b5284430820 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 12 Oct 2022 10:01:10 +0800 Subject: [PATCH 041/165] refactor: more rigorous checking of flapping to improve stability of the system port: https://github.com/emqx/emqx/pull/9045 --- apps/emqx/src/emqx_channel.erl | 16 +++++++--- apps/emqx/test/emqx_channel_SUITE.erl | 45 +++++++++++++++++++++------ 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 742868694..494c77ec7 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -345,7 +345,8 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) -> fun check_connect/2, fun enrich_client/2, fun set_log_meta/2, - fun check_banned/2 + fun check_banned/2, + fun count_flapping_event/2 ], ConnPkt, Channel#channel{conn_state = connecting} @@ -1260,14 +1261,11 @@ handle_info( {sock_closed, Reason}, Channel = #channel{ - conn_state = ConnState, - clientinfo = ClientInfo = #{zone := Zone} + conn_state = ConnState } ) when ConnState =:= connected orelse ConnState =:= reauthenticating -> - emqx_config:get_zone_conf(Zone, [flapping_detect, enable]) andalso - emqx_flapping:detect(ClientInfo), Channel1 = ensure_disconnected(Reason, maybe_publish_will_msg(Channel)), case maybe_shutdown(Reason, Channel1) of {ok, Channel2} -> {ok, {event, disconnected}, Channel2}; @@ -1636,6 +1634,14 @@ check_banned(_ConnPkt, #channel{clientinfo = ClientInfo}) -> false -> ok end. +%%-------------------------------------------------------------------- +%% Flapping + +count_flapping_event(_ConnPkt, Channel = #channel{clientinfo = ClientInfo = #{zone := Zone}}) -> + emqx_config:get_zone_conf(Zone, [flapping_detect, enable]) andalso + emqx_flapping:detect(ClientInfo), + {ok, Channel}. + %%-------------------------------------------------------------------- %% Authenticate diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index df1720772..a3fa3e5bc 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -207,14 +207,6 @@ init_per_suite(Config) -> ok = meck:new(emqx_cm, [passthrough, no_history, no_link]), ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end), ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end), - %% Access Control Meck - ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), - ok = meck:expect( - emqx_access_control, - authenticate, - fun(_) -> {ok, #{is_superuser => false}} end - ), - ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end), %% Broker Meck ok = meck:new(emqx_broker, [passthrough, no_history, no_link]), %% Hooks Meck @@ -234,7 +226,6 @@ init_per_suite(Config) -> end_per_suite(_Config) -> meck:unload([ - emqx_access_control, emqx_metrics, emqx_session, emqx_broker, @@ -244,11 +235,21 @@ end_per_suite(_Config) -> ]). init_per_testcase(_TestCase, Config) -> + %% Access Control Meck + ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), + ok = meck:expect( + emqx_access_control, + authenticate, + fun(_) -> {ok, #{is_superuser => false}} end + ), + ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end), + %% Set confs OldConf = set_test_listener_confs(), emqx_common_test_helpers:start_apps([]), [{config, OldConf} | Config]. end_per_testcase(_TestCase, Config) -> + meck:unload([emqx_access_control]), emqx_config:put(?config(config, Config)), emqx_common_test_helpers:stop_apps([]), Config. @@ -1115,6 +1116,32 @@ t_ws_cookie_init(_) -> ), ?assertMatch(#{ws_cookie := WsCookie}, emqx_channel:info(clientinfo, Channel)). +%%-------------------------------------------------------------------- +%% Test cases for other mechnisms +%%-------------------------------------------------------------------- + +t_flapping_detect(_) -> + emqx_config:put_zone_conf(default, [flapping_detect, enable], true), + Parent = self(), + ok = meck:expect( + emqx_cm, + open_session, + fun(true, _ClientInfo, _ConnInfo) -> + {ok, #{session => session(), present => false}} + end + ), + ok = meck:expect(emqx_access_control, authenticate, fun(_) -> {error, not_authorized} end), + ok = meck:expect(emqx_flapping, detect, fun(_) -> Parent ! flapping_detect end), + IdleChannel = channel(#{conn_state => idle}), + {shutdown, not_authorized, _ConnAck, _Channel} = + emqx_channel:handle_in(?CONNECT_PACKET(connpkt()), IdleChannel), + receive + flapping_detect -> ok + after 2000 -> + ?assert(false, "Flapping detect should be exected in connecting progress") + end, + meck:unload([emqx_flapping]). + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- From 59c1b0d16e8b303bdef6eba688145bf92db0bc04 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 12 Oct 2022 10:04:20 +0800 Subject: [PATCH 042/165] chore: update changes --- CHANGES-5.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 9cc2d508c..8d3c1e61d 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -4,6 +4,7 @@ * Add `cert_common_name` and `cert_subject` placeholder support for authz_http and authz_mongo.[#8973](https://github.com/emqx/emqx/pull/8973) * Use milliseconds internally in emqx_delayed to store the publish time, improving precision.[#9060](https://github.com/emqx/emqx/pull/9060) +* More rigorous checking of flapping to improve stability of the system. [#9136](https://github.com/emqx/emqx/pull/9136) ## Bug fixes From 4579d7fff8ad70c4d93f3d34edf7531fd98bb878 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Oct 2022 11:26:41 +0800 Subject: [PATCH 043/165] feat: get trace file's detail via /trace/:name/log_detail --- CHANGES-5.0.md | 1 + apps/emqx/src/emqx_trace/emqx_trace.erl | 12 ++++ .../src/emqx_mgmt_api_trace.erl | 67 +++++++++++++++++-- .../src/proto/emqx_mgmt_trace_proto_v2.erl | 66 ++++++++++++++++++ .../test/emqx_mgmt_api_trace_SUITE.erl | 8 ++- 5 files changed, 146 insertions(+), 8 deletions(-) create mode 100644 apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 8d3c1e61d..34356719b 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -5,6 +5,7 @@ * Add `cert_common_name` and `cert_subject` placeholder support for authz_http and authz_mongo.[#8973](https://github.com/emqx/emqx/pull/8973) * Use milliseconds internally in emqx_delayed to store the publish time, improving precision.[#9060](https://github.com/emqx/emqx/pull/9060) * More rigorous checking of flapping to improve stability of the system. [#9136](https://github.com/emqx/emqx/pull/9136) +* Add /trace/:name/log_detail HTTP API to return trace file's size and mtime. ## Bug fixes diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index e9866fb16..65756fc2f 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -19,6 +19,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("kernel/include/file.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). -export([ @@ -46,6 +47,7 @@ filename/2, trace_dir/0, trace_file/1, + trace_file_detail/1, delete_files_after_send/2 ]). @@ -193,6 +195,16 @@ trace_file(File) -> {error, Reason} -> {error, Node, Reason} end. +trace_file_detail(File) -> + FileName = filename:join(trace_dir(), File), + Node = atom_to_binary(node()), + case file:read_file_info(FileName, [{'time', 'posix'}]) of + {ok, #file_info{size = Size, mtime = Mtime}} -> + {ok, #{size => Size, mtime => Mtime, node => Node}}; + {error, Reason} -> + {error, #{reason => Reason, node => Node, file => File}} + end. + delete_files_after_send(TraceLog, Zips) -> gen_server:cast(?MODULE, {delete_tag, self(), [TraceLog | Zips]}). diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 7a9ae5710..31a8635cd 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -34,7 +34,8 @@ delete_trace/2, update_trace/2, download_trace_log/2, - stream_log_file/2 + stream_log_file/2, + log_file_detail/2 ]). -export([validate_name/1]). @@ -55,7 +56,14 @@ api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}). paths() -> - ["/trace", "/trace/:name/stop", "/trace/:name/download", "/trace/:name/log", "/trace/:name"]. + [ + "/trace", + "/trace/:name/stop", + "/trace/:name/download", + "/trace/:name/log", + "/trace/:name/log_detail", + "/trace/:name" + ]. schema("/trace") -> #{ @@ -95,7 +103,7 @@ schema("/trace/:name") -> #{ 'operationId' => delete_trace, delete => #{ - description => "Delete trace by name", + description => "Delete specified trace", tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -136,6 +144,19 @@ schema("/trace/:name/download") -> } } }; +schema("/trace/:name/log_detail") -> + #{ + 'operationId' => log_file_detail, + get => #{ + description => "get trace log file's detail message, such as size, last update time", + tags => ?TAGS, + parameters => [hoconsc:ref(name)], + responses => #{ + 200 => hoconsc:array(hoconsc:ref(log_file_detail)), + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Trace Name Not Found">>) + } + } + }; schema("/trace/:name/log") -> #{ 'operationId' => stream_log_file, @@ -158,6 +179,13 @@ schema("/trace/:name/log") -> } }. +fields(log_file_detail) -> + fields(node) ++ + [ + {size, hoconsc:mk(integer(), #{desc => "file size"})}, + {mtime, + hoconsc:mk(integer(), #{desc => "the modification and last access times of a file"})} + ]; fields(trace) -> [ {name, @@ -265,7 +293,8 @@ fields(node) -> #{ desc => "Node name", in => query, - required => false + required => false, + example => "emqx@127.0.0.1" } )} ]; @@ -323,7 +352,7 @@ trace(get, _Params) -> emqx_trace:format(List0) ), Nodes = mria_mnesia:running_nodes(), - TraceSize = wrap_rpc(emqx_mgmt_trace_proto_v1:get_trace_size(Nodes)), + TraceSize = wrap_rpc(emqx_mgmt_trace_proto_v2:get_trace_size(Nodes)), AllFileSize = lists:foldl(fun(F, Acc) -> maps:merge(Acc, F) end, #{}, TraceSize), Now = erlang:system_time(second), Traces = @@ -471,19 +500,43 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) -> ). collect_trace_file(Nodes, TraceLog) -> - wrap_rpc(emqx_mgmt_trace_proto_v1:trace_file(Nodes, TraceLog)). + wrap_rpc(emqx_mgmt_trace_proto_v2:trace_file(Nodes, TraceLog)). + +collect_trace_file_detail(TraceLog) -> + Nodes = mria_mnesia:running_nodes(), + wrap_rpc(emqx_mgmt_trace_proto_v2:trace_file_detail(Nodes, TraceLog)). wrap_rpc({GoodRes, BadNodes}) -> BadNodes =/= [] andalso ?SLOG(error, #{msg => "rpc_call_failed", bad_nodes => BadNodes}), GoodRes. +log_file_detail(get, #{bindings := #{name := Name}}) -> + case emqx_trace:get_trace_filename(Name) of + {ok, TraceLog} -> + TraceFiles = collect_trace_file_detail(TraceLog), + {200, group_trace_file_detail(TraceFiles)}; + {error, not_found} -> + ?NOT_FOUND(Name) + end. + +group_trace_file_detail(TraceLogDetail) -> + GroupFun = + fun + ({ok, Info}, Acc) -> + [Info | Acc]; + ({error, Error}, Acc) -> + ?SLOG(error, Error#{msg => "read_trace_file_failed"}), + Acc + end, + lists:foldl(GroupFun, [], TraceLogDetail). + stream_log_file(get, #{bindings := #{name := Name}, query_string := Query}) -> Position = maps:get(<<"position">>, Query, 0), Bytes = maps:get(<<"bytes">>, Query, 1000), case parse_node(Query, node()) of {ok, Node} -> - case emqx_mgmt_trace_proto_v1:read_trace_file(Node, Name, Position, Bytes) of + case emqx_mgmt_trace_proto_v2:read_trace_file(Node, Name, Position, Bytes) of {ok, Bin} -> Meta = #{<<"position">> => Position + byte_size(Bin), <<"bytes">> => Bytes}, {200, #{meta => Meta, items => Bin}}; diff --git a/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl b/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl new file mode 100644 index 000000000..a409c5f15 --- /dev/null +++ b/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl @@ -0,0 +1,66 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mgmt_trace_proto_v2). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + trace_file/2, + trace_file_detail/2, + get_trace_size/1, + read_trace_file/4 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.9". + +-spec get_trace_size([node()]) -> + emqx_rpc:multicall_result(#{{node(), file:name_all()} => non_neg_integer()}). +get_trace_size(Nodes) -> + rpc:multicall(Nodes, emqx_mgmt_api_trace, get_trace_size, [], 30000). + +-spec trace_file([node()], file:name_all()) -> + emqx_rpc:multicall_result( + {ok, Node :: list(), Binary :: binary()} + | {error, Node :: list(), Reason :: term()} + ). +trace_file(Nodes, File) -> + rpc:multicall(Nodes, emqx_trace, trace_file, [File], 60000). + +-spec trace_file_detail([node()], file:name_all()) -> + emqx_rpc:multicall_result( + {ok, #{ + size => non_neg_integer(), + mtime => file:date_time() | non_neg_integer() | 'undefined', + node => atom() + }} + | {error, #{reason => term(), node => atom(), file => file:name_all()}} + ). +trace_file_detail(Nodes, File) -> + rpc:multicall(Nodes, emqx_trace, trace_file_detail, [File], 25000). + +-spec read_trace_file(node(), binary(), non_neg_integer(), non_neg_integer()) -> + {ok, binary()} + | {error, _} + | {eof, non_neg_integer()} + | {badrpc, _}. +read_trace_file(Node, Name, Position, Limit) -> + rpc:call(Node, emqx_mgmt_api_trace, read_trace_file, [Name, Position, Limit]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index 5130913c1..72737ba60 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -175,7 +175,7 @@ t_create_failed(_Config) -> emqx_trace:clear(), ok. -t_download_log(_Config) -> +t_log_file(_Config) -> ClientId = <<"client-test-download">>, Now = erlang:system_time(second), Name = <<"test_client_id">>, @@ -191,6 +191,12 @@ t_download_log(_Config) -> ], ok = emqx_trace_handler_SUITE:filesync(Name, clientid), Header = auth_header_(), + ?assertMatch( + {error, {"HTTP/1.1", 404, "Not Found"}, _}, + request_api(get, api_path("trace/test_client_not_found/log_detail"), Header) + ), + {ok, Detail} = request_api(get, api_path("trace/test_client_id/log_detail"), Header), + ?assertMatch([#{<<"mtime">> := _, <<"size">> := _, <<"node">> := _}], json(Detail)), {ok, Binary} = request_api(get, api_path("trace/test_client_id/download"), Header), {ok, [ _Comment, From a02fea0a19868b9d91008f631272abe39d613f23 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Oct 2022 15:16:09 +0800 Subject: [PATCH 044/165] chore: update bpapi.verions --- apps/emqx/priv/bpapi.versions | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index e224e6425..062fe084d 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -20,6 +20,7 @@ {emqx_mgmt_api_plugins,1}. {emqx_mgmt_cluster,1}. {emqx_mgmt_trace,1}. +{emqx_mgmt_trace,2}. {emqx_persistent_session,1}. {emqx_plugin_libs,1}. {emqx_prometheus,1}. From c2f176c1e717faf2b6eae63b77cc9aa9948a3153 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 14 Oct 2022 09:49:46 +0200 Subject: [PATCH 045/165] fix(api): publish API only returns message ID --- .../src/emqx_mgmt_api_publish.erl | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_publish.erl b/apps/emqx_management/src/emqx_mgmt_api_publish.erl index bd214a87c..a8dfc052e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_publish.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_publish.erl @@ -109,15 +109,15 @@ fields(publish_message_info) -> [ {id, hoconsc:mk(binary(), #{ - desc => <<"Internal Message ID">> + desc => <<"A globally unique message ID">> })} - ] ++ fields(message). + ]. publish(post, #{body := Body}) -> case message(Body) of {ok, Message} -> _ = emqx_mgmt:publish(Message), - {200, format_message(Message)}; + {200, format_message_response(Message)}; {error, R} -> {400, 'BAD_REQUEST', to_binary(R)} end. @@ -126,7 +126,7 @@ publish_batch(post, #{body := Body}) -> case messages(Body) of {ok, Messages} -> _ = [emqx_mgmt:publish(Message) || Message <- Messages], - {200, format_message(Messages)}; + {200, format_message_response(Messages)}; {error, R} -> {400, 'BAD_REQUEST', to_binary(R)} end. @@ -167,19 +167,10 @@ messages([MessageMap | List], Res) -> {error, R} end. -format_message(Messages) when is_list(Messages) -> - [format_message(Message) || Message <- Messages]; -format_message(#message{ - id = ID, qos = Qos, from = From, topic = Topic, payload = Payload, flags = Flags -}) -> - #{ - id => emqx_guid:to_hexstr(ID), - qos => Qos, - topic => Topic, - payload => Payload, - retain => maps:get(retain, Flags, false), - clientid => to_binary(From) - }. +format_message_response(Messages) when is_list(Messages) -> + [format_message_response(Message) || Message <- Messages]; +format_message_response(#message{id = ID}) -> + #{id => emqx_guid:to_hexstr(ID)}. to_binary(Data) when is_binary(Data) -> Data; From 53fb8fcded470662261377c39f977e87c3e3f4c3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 14 Oct 2022 10:05:04 +0200 Subject: [PATCH 046/165] docs: Add changelog --- CHANGES-5.0.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 8d3c1e61d..d75766f0b 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -5,6 +5,9 @@ * Add `cert_common_name` and `cert_subject` placeholder support for authz_http and authz_mongo.[#8973](https://github.com/emqx/emqx/pull/8973) * Use milliseconds internally in emqx_delayed to store the publish time, improving precision.[#9060](https://github.com/emqx/emqx/pull/9060) * More rigorous checking of flapping to improve stability of the system. [#9136](https://github.com/emqx/emqx/pull/9136) +* No message(s) echo for the message publish APIs [#9155](https://github.com/emqx/emqx/pull/9155) + Prior to this fix, the message publish APIs (`api/v5/publish` and `api/v5/publish/bulk`) echos the message back to the client in HTTP body. + This change fixed it to only send back the message ID. ## Bug fixes From 5a8cab46a956f275c6daf38951f999c5dce41e89 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 14 Oct 2022 10:17:22 +0200 Subject: [PATCH 047/165] test: update publish api test suite --- .../test/emqx_mgmt_api_publish_SUITE.erl | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl index 4d17451c9..b4b3aa902 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl @@ -47,8 +47,10 @@ t_publish_api(_) -> Path = emqx_mgmt_api_test_util:api_path(["publish"]), Auth = emqx_mgmt_api_test_util:auth_header_(), Body = #{topic => ?TOPIC1, payload => Payload}, - {ok, _} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body), - ?assertEqual(receive_assert(?TOPIC1, 0, Payload), ok), + {ok, Response} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body), + ResponseMap = emqx_json:decode(Response, [return_maps]), + ?assertEqual([<<"id">>], maps:keys(ResponseMap)), + ?assertEqual(ok, receive_assert(?TOPIC1, 0, Payload)), emqtt:disconnect(Client). t_publish_bulk_api(_) -> @@ -63,10 +65,16 @@ t_publish_bulk_api(_) -> Auth = emqx_mgmt_api_test_util:auth_header_(), Body = [#{topic => ?TOPIC1, payload => Payload}, #{topic => ?TOPIC2, payload => Payload}], {ok, Response} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body), - ResponseMap = emqx_json:decode(Response, [return_maps]), - ?assertEqual(2, erlang:length(ResponseMap)), - ?assertEqual(receive_assert(?TOPIC1, 0, Payload), ok), - ?assertEqual(receive_assert(?TOPIC2, 0, Payload), ok), + ResponseList = emqx_json:decode(Response, [return_maps]), + ?assertEqual(2, erlang:length(ResponseList)), + lists:foreach( + fun(ResponseMap) -> + ?assertEqual([<<"id">>], maps:keys(ResponseMap)) + end, + ResponseList + ), + ?assertEqual(ok, receive_assert(?TOPIC1, 0, Payload)), + ?assertEqual(ok, receive_assert(?TOPIC2, 0, Payload)), emqtt:disconnect(Client). receive_assert(Topic, Qos, Payload) -> From f2db35db3926f1e975329a2fe0c3f4eeb1a4412f Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 14 Oct 2022 10:46:11 +0200 Subject: [PATCH 048/165] chore: delete unused function clause --- apps/emqx_management/src/emqx_mgmt_api_publish.erl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_publish.erl b/apps/emqx_management/src/emqx_mgmt_api_publish.erl index a8dfc052e..0e05a3875 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_publish.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_publish.erl @@ -172,7 +172,5 @@ format_message_response(Messages) when is_list(Messages) -> format_message_response(#message{id = ID}) -> #{id => emqx_guid:to_hexstr(ID)}. -to_binary(Data) when is_binary(Data) -> - Data; -to_binary(Data) -> - list_to_binary(io_lib:format("~p", [Data])). +to_binary(Term) -> + list_to_binary(io_lib:format("~p", [Term])). From 37a400ec25cf9b540b222f8eb64fcbc52f01cb34 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 14 Oct 2022 11:19:25 +0200 Subject: [PATCH 049/165] chore: bump asdf erlang version --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index 0b6e665c1..0f7c9b32e 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -erlang 24.2.1-1 +erlang 24.3.4.2-1 elixir 1.13.4-otp-24 From 4d9cae03875c59c6f1ff2cf690aafe47332d8918 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Oct 2022 11:26:41 +0800 Subject: [PATCH 050/165] feat: get trace file's detail via /trace/:name/log_detail --- CHANGES-5.0.md | 1 + apps/emqx/src/emqx_trace/emqx_trace.erl | 12 ++++ .../src/emqx_mgmt_api_trace.erl | 67 +++++++++++++++++-- .../src/proto/emqx_mgmt_trace_proto_v2.erl | 66 ++++++++++++++++++ .../test/emqx_mgmt_api_trace_SUITE.erl | 8 ++- 5 files changed, 146 insertions(+), 8 deletions(-) create mode 100644 apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index a1171a01e..0f617678d 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -8,6 +8,7 @@ * No message(s) echo for the message publish APIs [#9155](https://github.com/emqx/emqx/pull/9155) Prior to this fix, the message publish APIs (`api/v5/publish` and `api/v5/publish/bulk`) echos the message back to the client in HTTP body. This change fixed it to only send back the message ID. +* Add /trace/:name/log_detail HTTP API to return trace file's size and mtime [#9152](https://github.com/emqx/emqx/pull/9152) ## Bug fixes diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index e9866fb16..65756fc2f 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -19,6 +19,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("kernel/include/file.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). -export([ @@ -46,6 +47,7 @@ filename/2, trace_dir/0, trace_file/1, + trace_file_detail/1, delete_files_after_send/2 ]). @@ -193,6 +195,16 @@ trace_file(File) -> {error, Reason} -> {error, Node, Reason} end. +trace_file_detail(File) -> + FileName = filename:join(trace_dir(), File), + Node = atom_to_binary(node()), + case file:read_file_info(FileName, [{'time', 'posix'}]) of + {ok, #file_info{size = Size, mtime = Mtime}} -> + {ok, #{size => Size, mtime => Mtime, node => Node}}; + {error, Reason} -> + {error, #{reason => Reason, node => Node, file => File}} + end. + delete_files_after_send(TraceLog, Zips) -> gen_server:cast(?MODULE, {delete_tag, self(), [TraceLog | Zips]}). diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 7a9ae5710..31a8635cd 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -34,7 +34,8 @@ delete_trace/2, update_trace/2, download_trace_log/2, - stream_log_file/2 + stream_log_file/2, + log_file_detail/2 ]). -export([validate_name/1]). @@ -55,7 +56,14 @@ api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}). paths() -> - ["/trace", "/trace/:name/stop", "/trace/:name/download", "/trace/:name/log", "/trace/:name"]. + [ + "/trace", + "/trace/:name/stop", + "/trace/:name/download", + "/trace/:name/log", + "/trace/:name/log_detail", + "/trace/:name" + ]. schema("/trace") -> #{ @@ -95,7 +103,7 @@ schema("/trace/:name") -> #{ 'operationId' => delete_trace, delete => #{ - description => "Delete trace by name", + description => "Delete specified trace", tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -136,6 +144,19 @@ schema("/trace/:name/download") -> } } }; +schema("/trace/:name/log_detail") -> + #{ + 'operationId' => log_file_detail, + get => #{ + description => "get trace log file's detail message, such as size, last update time", + tags => ?TAGS, + parameters => [hoconsc:ref(name)], + responses => #{ + 200 => hoconsc:array(hoconsc:ref(log_file_detail)), + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Trace Name Not Found">>) + } + } + }; schema("/trace/:name/log") -> #{ 'operationId' => stream_log_file, @@ -158,6 +179,13 @@ schema("/trace/:name/log") -> } }. +fields(log_file_detail) -> + fields(node) ++ + [ + {size, hoconsc:mk(integer(), #{desc => "file size"})}, + {mtime, + hoconsc:mk(integer(), #{desc => "the modification and last access times of a file"})} + ]; fields(trace) -> [ {name, @@ -265,7 +293,8 @@ fields(node) -> #{ desc => "Node name", in => query, - required => false + required => false, + example => "emqx@127.0.0.1" } )} ]; @@ -323,7 +352,7 @@ trace(get, _Params) -> emqx_trace:format(List0) ), Nodes = mria_mnesia:running_nodes(), - TraceSize = wrap_rpc(emqx_mgmt_trace_proto_v1:get_trace_size(Nodes)), + TraceSize = wrap_rpc(emqx_mgmt_trace_proto_v2:get_trace_size(Nodes)), AllFileSize = lists:foldl(fun(F, Acc) -> maps:merge(Acc, F) end, #{}, TraceSize), Now = erlang:system_time(second), Traces = @@ -471,19 +500,43 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) -> ). collect_trace_file(Nodes, TraceLog) -> - wrap_rpc(emqx_mgmt_trace_proto_v1:trace_file(Nodes, TraceLog)). + wrap_rpc(emqx_mgmt_trace_proto_v2:trace_file(Nodes, TraceLog)). + +collect_trace_file_detail(TraceLog) -> + Nodes = mria_mnesia:running_nodes(), + wrap_rpc(emqx_mgmt_trace_proto_v2:trace_file_detail(Nodes, TraceLog)). wrap_rpc({GoodRes, BadNodes}) -> BadNodes =/= [] andalso ?SLOG(error, #{msg => "rpc_call_failed", bad_nodes => BadNodes}), GoodRes. +log_file_detail(get, #{bindings := #{name := Name}}) -> + case emqx_trace:get_trace_filename(Name) of + {ok, TraceLog} -> + TraceFiles = collect_trace_file_detail(TraceLog), + {200, group_trace_file_detail(TraceFiles)}; + {error, not_found} -> + ?NOT_FOUND(Name) + end. + +group_trace_file_detail(TraceLogDetail) -> + GroupFun = + fun + ({ok, Info}, Acc) -> + [Info | Acc]; + ({error, Error}, Acc) -> + ?SLOG(error, Error#{msg => "read_trace_file_failed"}), + Acc + end, + lists:foldl(GroupFun, [], TraceLogDetail). + stream_log_file(get, #{bindings := #{name := Name}, query_string := Query}) -> Position = maps:get(<<"position">>, Query, 0), Bytes = maps:get(<<"bytes">>, Query, 1000), case parse_node(Query, node()) of {ok, Node} -> - case emqx_mgmt_trace_proto_v1:read_trace_file(Node, Name, Position, Bytes) of + case emqx_mgmt_trace_proto_v2:read_trace_file(Node, Name, Position, Bytes) of {ok, Bin} -> Meta = #{<<"position">> => Position + byte_size(Bin), <<"bytes">> => Bytes}, {200, #{meta => Meta, items => Bin}}; diff --git a/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl b/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl new file mode 100644 index 000000000..a409c5f15 --- /dev/null +++ b/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl @@ -0,0 +1,66 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mgmt_trace_proto_v2). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + trace_file/2, + trace_file_detail/2, + get_trace_size/1, + read_trace_file/4 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.9". + +-spec get_trace_size([node()]) -> + emqx_rpc:multicall_result(#{{node(), file:name_all()} => non_neg_integer()}). +get_trace_size(Nodes) -> + rpc:multicall(Nodes, emqx_mgmt_api_trace, get_trace_size, [], 30000). + +-spec trace_file([node()], file:name_all()) -> + emqx_rpc:multicall_result( + {ok, Node :: list(), Binary :: binary()} + | {error, Node :: list(), Reason :: term()} + ). +trace_file(Nodes, File) -> + rpc:multicall(Nodes, emqx_trace, trace_file, [File], 60000). + +-spec trace_file_detail([node()], file:name_all()) -> + emqx_rpc:multicall_result( + {ok, #{ + size => non_neg_integer(), + mtime => file:date_time() | non_neg_integer() | 'undefined', + node => atom() + }} + | {error, #{reason => term(), node => atom(), file => file:name_all()}} + ). +trace_file_detail(Nodes, File) -> + rpc:multicall(Nodes, emqx_trace, trace_file_detail, [File], 25000). + +-spec read_trace_file(node(), binary(), non_neg_integer(), non_neg_integer()) -> + {ok, binary()} + | {error, _} + | {eof, non_neg_integer()} + | {badrpc, _}. +read_trace_file(Node, Name, Position, Limit) -> + rpc:call(Node, emqx_mgmt_api_trace, read_trace_file, [Name, Position, Limit]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index 5130913c1..72737ba60 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -175,7 +175,7 @@ t_create_failed(_Config) -> emqx_trace:clear(), ok. -t_download_log(_Config) -> +t_log_file(_Config) -> ClientId = <<"client-test-download">>, Now = erlang:system_time(second), Name = <<"test_client_id">>, @@ -191,6 +191,12 @@ t_download_log(_Config) -> ], ok = emqx_trace_handler_SUITE:filesync(Name, clientid), Header = auth_header_(), + ?assertMatch( + {error, {"HTTP/1.1", 404, "Not Found"}, _}, + request_api(get, api_path("trace/test_client_not_found/log_detail"), Header) + ), + {ok, Detail} = request_api(get, api_path("trace/test_client_id/log_detail"), Header), + ?assertMatch([#{<<"mtime">> := _, <<"size">> := _, <<"node">> := _}], json(Detail)), {ok, Binary} = request_api(get, api_path("trace/test_client_id/download"), Header), {ok, [ _Comment, From 58b990064c3edcda260289fc50cf6b19623ed3ea Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Oct 2022 15:16:09 +0800 Subject: [PATCH 051/165] chore: update bpapi.verions --- apps/emqx/priv/bpapi.versions | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index e224e6425..062fe084d 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -20,6 +20,7 @@ {emqx_mgmt_api_plugins,1}. {emqx_mgmt_cluster,1}. {emqx_mgmt_trace,1}. +{emqx_mgmt_trace,2}. {emqx_persistent_session,1}. {emqx_plugin_libs,1}. {emqx_prometheus,1}. From 77a5125366173e7f6c7770461d4652b388002573 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 17 Oct 2022 09:45:00 +0800 Subject: [PATCH 052/165] chore: update apps/emqx_management/src/emqx_mgmt_api_trace.erl Co-authored-by: Thales Macedo Garitezi --- apps/emqx_management/src/emqx_mgmt_api_trace.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 31a8635cd..587257688 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -148,7 +148,7 @@ schema("/trace/:name/log_detail") -> #{ 'operationId' => log_file_detail, get => #{ - description => "get trace log file's detail message, such as size, last update time", + description => "get trace log file's metadata, such as size, last update time", tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ From 228118b1faa26d85ab3af2f89104dc8a821ec7ea Mon Sep 17 00:00:00 2001 From: Tautcius Date: Fri, 14 Oct 2022 08:11:24 +0300 Subject: [PATCH 053/165] chore(charts): Update certificate.yaml Typo in if statement that prevented creation of Certificate resource. --- deploy/charts/emqx/templates/certificate.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/charts/emqx/templates/certificate.yaml b/deploy/charts/emqx/templates/certificate.yaml index 36b7f6521..9a2ed969a 100644 --- a/deploy/charts/emqx/templates/certificate.yaml +++ b/deploy/charts/emqx/templates/certificate.yaml @@ -1,4 +1,4 @@ -{{- if and (.Values.ssl.enable) (not .Values.ssl.useExisting) -}} +{{- if and (.Values.ssl.enabled) (not .Values.ssl.useExisting) -}} --- apiVersion: cert-manager.io/v1 kind: Certificate From 69701ff578adf2085f32cd4af484e725ea704123 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 12 Oct 2022 14:13:11 +0800 Subject: [PATCH 054/165] feat(banned): clean retained/delayed data when client is banned --- apps/emqx/i18n/emqx_schema_i18n.conf | 13 ++++ apps/emqx/priv/bpapi.versions | 2 +- apps/emqx/src/emqx_banned.erl | 54 +++++++++++++---- apps/emqx/src/emqx_flapping.erl | 4 +- apps/emqx/src/emqx_schema.erl | 8 +++ apps/emqx/test/emqx_flapping_SUITE.erl | 3 +- .../i18n/emqx_mgmt_api_banned_i18n.conf | 12 ++++ .../src/emqx_mgmt_api_banned.erl | 9 ++- apps/emqx_modules/src/emqx_delayed.erl | 31 ++++++++-- .../src/proto/emqx_delayed_proto_v2.erl | 45 ++++++++++++++ apps/emqx_modules/test/emqx_delayed_SUITE.erl | 59 ++++++++++++++++++ apps/emqx_retainer/src/emqx_retainer.erl | 24 +++++++- .../src/emqx_retainer_mnesia.erl | 33 +++++++--- .../test/emqx_retainer_SUITE.erl | 60 +++++++++++++++++++ 14 files changed, 325 insertions(+), 32 deletions(-) create mode 100644 apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 17e59a4dd..8f30d3c41 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -495,6 +495,19 @@ emqx_schema { } } + flapping_detect_clean_when_banned { + desc { + en: "Clean retained/delayed messages when client is banned.\n" + "Note: This may be expensive and only supports users banned by clientid." + zh: "当客户端被禁时删除其保留、延迟消息" + "注意: 这个操作开销可能较大,且只支持通过 clientid 封禁的用户数据。" + } + label: { + en: "Clean when banned" + zh: "被禁时清理消息" + } + } + persistent_session_store_enabled { desc { en: "Use the database to store information about persistent sessions.\n" diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 062fe084d..f5c515cce 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -8,7 +8,7 @@ {emqx_conf,1}. {emqx_conf,2}. {emqx_dashboard,1}. -{emqx_delayed,1}. +{emqx_delayed,2}. {emqx_exhook,1}. {emqx_gateway_api_listeners,1}. {emqx_gateway_cm,1}. diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl index cf81c735b..d7443543f 100644 --- a/apps/emqx/src/emqx_banned.erl +++ b/apps/emqx/src/emqx_banned.erl @@ -32,11 +32,13 @@ -export([ check/1, create/1, + create/2, look_up/1, delete/1, info/1, format/1, - parse/1 + parse/1, + parse_opts/1 ]). %% gen_server callbacks @@ -63,6 +65,13 @@ -compile(nowarn_export_all). -endif. +-type banned_opts() :: #{ + clean => boolean(), + atom() => term() +}. + +-export_type([banned_opts/0]). + %%-------------------------------------------------------------------- %% Mnesia bootstrap %%-------------------------------------------------------------------- @@ -141,6 +150,11 @@ parse(Params) -> {error, ErrorReason} end end. + +parse_opts(Params) -> + Clean = maps:get(<<"clean">>, Params, false), + #{clean => Clean}. + pares_who(#{as := As, who := Who}) -> pares_who(#{<<"as">> => As, <<"who">> => Who}); pares_who(#{<<"as">> := peerhost, <<"who">> := Peerhost0}) -> @@ -162,13 +176,15 @@ to_rfc3339(Timestamp) -> -spec create(emqx_types:banned() | map()) -> {ok, emqx_types:banned()} | {error, {already_exist, emqx_types:banned()}}. -create(#{ - who := Who, - by := By, - reason := Reason, - at := At, - until := Until -}) -> +create( + #{ + who := Who, + by := By, + reason := Reason, + at := At, + until := Until + } = Data +) -> Banned = #banned{ who = Who, by = By, @@ -176,11 +192,16 @@ create(#{ at = At, until = Until }, - create(Banned); -create(Banned = #banned{who = Who}) -> + create(Banned, Data); +create(Banned = #banned{}) -> + create(Banned, #{clean => false}). + +-spec create(emqx_types:banned(), banned_opts()) -> + {ok, emqx_types:banned()} | {error, {already_exist, emqx_types:banned()}}. +create(Banned = #banned{who = Who}, Opts) -> case look_up(Who) of [] -> - mria:dirty_write(?BANNED_TAB, Banned), + insert_banned(Banned, Opts), {ok, Banned}; [OldBanned = #banned{until = Until}] -> %% Don't support shorten or extend the until time by overwrite. @@ -190,7 +211,7 @@ create(Banned = #banned{who = Who}) -> {error, {already_exist, OldBanned}}; %% overwrite expired one is ok. false -> - mria:dirty_write(?BANNED_TAB, Banned), + insert_banned(Banned, Opts), {ok, Banned} end end. @@ -266,3 +287,12 @@ expire_banned_items(Now) -> ok, ?BANNED_TAB ). + +insert_banned(Banned, Opts) -> + mria:dirty_write(?BANNED_TAB, Banned), + run_hooks(Banned, Opts). + +run_hooks(Banned, #{clean := true}) -> + emqx_hooks:run('client.banned', [Banned]); +run_hooks(_Banned, _Opts) -> + ok. diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index 7e72c488f..f0492cbf9 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -121,7 +121,7 @@ handle_cast( started_at = StartedAt, detect_cnt = DetectCnt }, - #{window_time := WindTime, ban_time := Interval}}, + #{window_time := WindTime, ban_time := Interval, clean_when_banned := Clean}}, State ) -> case now_diff(StartedAt) < WindTime of @@ -145,7 +145,7 @@ handle_cast( at = Now, until = Now + (Interval div 1000) }, - {ok, _} = emqx_banned:create(Banned), + {ok, _} = emqx_banned:create(Banned, #{clean => Clean}), ok; false -> ?SLOG( diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 135992965..17bd03088 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -640,6 +640,14 @@ fields("flapping_detect") -> default => "5m", desc => ?DESC(flapping_detect_ban_time) } + )}, + {"clean_when_banned", + sc( + boolean(), + #{ + default => false, + desc => ?DESC(flapping_detect_clean_when_banned) + } )} ]; fields("force_shutdown") -> diff --git a/apps/emqx/test/emqx_flapping_SUITE.erl b/apps/emqx/test/emqx_flapping_SUITE.erl index f37e20fdc..774ccaae2 100644 --- a/apps/emqx/test/emqx_flapping_SUITE.erl +++ b/apps/emqx/test/emqx_flapping_SUITE.erl @@ -34,7 +34,8 @@ init_per_suite(Config) -> % 0.1s window_time => 100, %% 2s - ban_time => 2000 + ban_time => 2000, + clean_when_banned => false } ), Config. diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf index 3045cb293..04b96891c 100644 --- a/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf +++ b/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf @@ -95,4 +95,16 @@ emqx_mgmt_api_banned { zh: """封禁结束时间""" } } + clean { + desc { + en: """Clean retained/delayed messages when client is banned.""" + """Note: This may be expensive and only supports users banned by clientid.""" + zh: """当客户端被禁时删除其保留、延迟消息""" + """注意: 这个操作开销可能较大,且只支持通过 clientid 封禁的用户数据。""" + } + label { + en: """Clean when banned""" + zh: """被禁时清理消息""" + } + } } diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 2eb8908c6..e3a99038c 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -150,6 +150,12 @@ fields(ban) -> desc => ?DESC(until), required => false, example => <<"2021-10-25T21:53:47+08:00">> + })}, + {clean, + hoconsc:mk(boolean(), #{ + desc => ?DESC(clean), + required => false, + example => false })} ]. @@ -161,7 +167,8 @@ banned(post, #{body := Body}) -> {error, Reason} -> {400, 'BAD_REQUEST', list_to_binary(Reason)}; Ban -> - case emqx_banned:create(Ban) of + Opts = emqx_banned:parse_opts(Body), + case emqx_banned:create(Ban, Opts) of {ok, Banned} -> {200, format(Banned)}; {error, {already_exist, Old}} -> diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 76646bc64..f703c9dce 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -23,6 +23,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). %% Mnesia bootstrap -export([mnesia/1]). @@ -31,7 +32,8 @@ -export([ start_link/0, - on_message_publish/1 + on_message_publish/1, + on_client_banned/1 ]). %% gen_server callbacks @@ -44,7 +46,7 @@ code_change/3 ]). -%% gen_server callbacks +%% API -export([ load/0, unload/0, @@ -57,7 +59,9 @@ delete_delayed_message/1, delete_delayed_message/2, cluster_list/1, - cluster_query/4 + cluster_query/4, + clean_by_clientid/1, + do_clean_by_clientid/1 ]). -export([ @@ -138,6 +142,11 @@ on_message_publish( on_message_publish(Msg) -> {ok, Msg}. +on_client_banned(#banned{who = {clientid, ClientId}}) -> + clean_by_clientid(ClientId); +on_client_banned(_) -> + ok. + %%-------------------------------------------------------------------- %% Start delayed publish server %%-------------------------------------------------------------------- @@ -228,7 +237,7 @@ get_delayed_message(Id) -> get_delayed_message(Node, Id) when Node =:= node() -> get_delayed_message(Id); get_delayed_message(Node, Id) -> - emqx_delayed_proto_v1:get_delayed_message(Node, Id). + emqx_delayed_proto_v2:get_delayed_message(Node, Id). -spec delete_delayed_message(binary()) -> with_id_return(). delete_delayed_message(Id) -> @@ -243,7 +252,7 @@ delete_delayed_message(Id) -> delete_delayed_message(Node, Id) when Node =:= node() -> delete_delayed_message(Id); delete_delayed_message(Node, Id) -> - emqx_delayed_proto_v1:delete_delayed_message(Node, Id). + emqx_delayed_proto_v2:delete_delayed_message(Node, Id). update_config(Config) -> emqx_conf:update([delayed], Config, #{rawconf_with_defaults => true, override_to => cluster}). @@ -252,6 +261,16 @@ post_config_update(_KeyPath, _ConfigReq, NewConf, _OldConf, _AppEnvs) -> Enable = maps:get(enable, NewConf, undefined), load_or_unload(Enable). +clean_by_clientid(ClientId) -> + Nodes = mria_mnesia:running_nodes(), + _ = [emqx_delayed_proto_v2:clean_by_clientid(Node, ClientId) || Node <- Nodes], + ok. + +do_clean_by_clientid(ClientId) -> + ets:select_delete( + ?TAB, ets:fun2ms(fun(#delayed_message{msg = Msg}) -> Msg#message.from =:= ClientId end) + ). + %%-------------------------------------------------------------------- %% gen_server callback %%-------------------------------------------------------------------- @@ -383,9 +402,11 @@ delayed_count() -> mnesia:table_info(?TAB, size). do_load_or_unload(true, State) -> emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_DELAY_PUB), + ok = emqx_hooks:put('client.banned', {?MODULE, on_client_banned, []}, ?HP_LOWEST), State; do_load_or_unload(false, #{publish_timer := PubTimer} = State) -> emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), + ok = emqx_hooks:del('client.banned', {?MODULE, on_client_banned}), emqx_misc:cancel_timer(PubTimer), ets:delete_all_objects(?TAB), State#{publish_timer := undefined, publish_at := 0}; diff --git a/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl b/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl new file mode 100644 index 000000000..bb09e47eb --- /dev/null +++ b/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl @@ -0,0 +1,45 @@ +%%-------------------------------------------------------------------- +%%Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_delayed_proto_v2). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + get_delayed_message/2, + delete_delayed_message/2, + clean_by_clientid/2 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.0". + +-spec get_delayed_message(node(), binary()) -> + emqx_delayed:with_id_return(map()) | emqx_rpc:badrpc(). +get_delayed_message(Node, Id) -> + rpc:call(Node, emqx_delayed, get_delayed_message, [Id]). + +-spec delete_delayed_message(node(), binary()) -> emqx_delayed:with_id_return() | emqx_rpc:badrpc(). +delete_delayed_message(Node, Id) -> + rpc:call(Node, emqx_delayed, delete_delayed_message, [Id]). + +-spec clean_by_clientid(node(), emqx_types:clientid()) -> + emqx_delayed:with_id_return() | emqx_rpc:badrpc(). +clean_by_clientid(Node, ClientID) -> + rpc:call(Node, emqx_delayed, do_clean_by_clientid, [ClientID]). diff --git a/apps/emqx_modules/test/emqx_delayed_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_SUITE.erl index e5e3db98c..f19d9388c 100644 --- a/apps/emqx_modules/test/emqx_delayed_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_SUITE.erl @@ -212,6 +212,65 @@ t_delayed_precision(_) -> _ = on_message_publish(DelayedMsg0), ?assert(FutureDiff() =< MaxSpan). +t_banned_clean(_) -> + emqx:update_config([delayed, max_delayed_messages], 10000), + ClientId1 = <<"bc1">>, + ClientId2 = <<"bc2">>, + {ok, C1} = emqtt:start_link([{clientid, ClientId1}, {clean_start, true}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C1), + + {ok, C2} = emqtt:start_link([{clientid, ClientId2}, {clean_start, true}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C2), + + [ + begin + emqtt:publish( + Conn, + <<"$delayed/60/0/", ClientId/binary>>, + <<"">>, + [{qos, 0}, {retain, false}] + ), + emqtt:publish( + Conn, + <<"$delayed/60/1/", ClientId/binary>>, + <<"">>, + [{qos, 0}, {retain, false}] + ) + end + || {ClientId, Conn} <- lists:zip([ClientId1, ClientId2], [C1, C2]) + ], + + emqtt:publish( + C2, + <<"$delayed/60/2/", ClientId2/binary>>, + <<"">>, + [{qos, 0}, {retain, false}] + ), + + timer:sleep(500), + ?assertMatch(#{meta := #{count := 5}}, emqx_delayed:list(#{page => 1, limit => 10})), + + Now = erlang:system_time(second), + Who = {clientid, ClientId2}, + emqx_banned:create(#{ + who => Who, + by => <<"test">>, + reason => <<"test">>, + at => Now, + until => Now + 120, + clean => true + }), + + timer:sleep(500), + + ?assertMatch(#{meta := #{count := 2}}, emqx_delayed:list(#{page => 1, limit => 10})), + + emqx_banned:delete(Who), + emqx_delayed:clean_by_clientid(ClientId1), + timer:sleep(500), + ok = emqtt:disconnect(C1), + ok = emqtt:disconnect(C2). + subscribe_proc() -> Self = self(), Ref = erlang:make_ref(), diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 5d911b5f4..d6a025a41 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -19,6 +19,7 @@ -behaviour(gen_server). -include("emqx_retainer.hrl"). +-include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). @@ -26,7 +27,8 @@ -export([ on_session_subscribed/4, - on_message_publish/2 + on_message_publish/2, + on_client_banned/1 ]). -export([ @@ -39,6 +41,7 @@ get_expiry_time/1, update_config/1, clean/0, + clean_by_clientid/1, delete/1, page_read/3, post_config_update/5, @@ -80,6 +83,7 @@ -callback match_messages(context(), topic(), cursor()) -> {ok, list(), cursor()}. -callback clear_expired(context()) -> ok. -callback clean(context()) -> ok. +-callback clean_by_clientid(context(), emqx_types:clientid()) -> ok. -callback size(context()) -> non_neg_integer(). %%-------------------------------------------------------------------- @@ -118,6 +122,11 @@ on_message_publish(Msg = #message{flags = #{retain := true}}, Context) -> on_message_publish(Msg, _) -> {ok, Msg}. +on_client_banned(#banned{who = {clientid, ClientId}}) -> + clean_by_clientid(ClientId); +on_client_banned(_) -> + ok. + %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -151,6 +160,9 @@ update_config(Conf) -> clean() -> call(?FUNCTION_NAME). +clean_by_clientid(ClientId) -> + call({?FUNCTION_NAME, ClientId}). + delete(Topic) -> call({?FUNCTION_NAME, Topic}). @@ -207,6 +219,9 @@ handle_call({update_config, NewConf, OldConf}, _, State) -> handle_call(clean, _, #{context := Context} = State) -> clean(Context), {reply, ok, State}; +handle_call({clean_by_clientid, ClientId}, _, #{context := Context} = State) -> + clean_by_clientid(Context, ClientId), + {reply, ok, State}; handle_call({delete, Topic}, _, #{context := Context} = State) -> delete_message(Context, Topic), {reply, ok, State}; @@ -298,6 +313,11 @@ clean(Context) -> Mod = get_backend_module(), Mod:clean(Context). +-spec clean_by_clientid(context(), emqx_types:clientid()) -> ok. +clean_by_clientid(Context, ClientId) -> + Mod = get_backend_module(), + Mod:clean_by_clientid(Context, ClientId). + -spec update_config(state(), hocons:config(), hocons:config()) -> state(). update_config(State, Conf, OldConf) -> update_config( @@ -433,11 +453,13 @@ load(Context) -> 'session.subscribed', {?MODULE, on_session_subscribed, [Context]}, ?HP_RETAINER ), ok = emqx_hooks:put('message.publish', {?MODULE, on_message_publish, [Context]}, ?HP_RETAINER), + ok = emqx_hooks:put('client.banned', {?MODULE, on_client_banned, []}, ?HP_LOWEST), emqx_stats:update_interval(emqx_retainer_stats, fun ?MODULE:stats_fun/0), ok. unload() -> ok = emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), ok = emqx_hooks:del('session.subscribed', {?MODULE, on_session_subscribed}), + ok = emqx_hooks:del('client.banned', {?MODULE, on_client_banned}), emqx_stats:cancel_update(emqx_retainer_stats), ok. diff --git a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl index c236b9c28..281d9c3ce 100644 --- a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl +++ b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl @@ -33,13 +33,14 @@ match_messages/3, clear_expired/1, clean/1, + clean_by_clientid/2, size/1 ]). %% Internal exports (RPC) -export([ do_store_retained/1, - do_clear_expired/0, + do_clear/1, do_delete_message/1, do_populate_index_meta/1, do_reindex_batch/2 @@ -61,6 +62,8 @@ -record(retained_index, {key, expiry_time}). -record(retained_index_meta, {key, read_indices, write_indices, reindexing, extra}). +-type retained_message() :: #retained_message{}. + -define(META_KEY, index_meta). -define(CLEAR_BATCH_SIZE, 1000). @@ -164,18 +167,22 @@ do_store_retained(#message{topic = Topic} = Msg) -> end. clear_expired(_) -> - {atomic, _} = mria:transaction(?RETAINER_SHARD, fun ?MODULE:do_clear_expired/0), + NowMs = erlang:system_time(millisecond), + {atomic, _} = mria:transaction(?RETAINER_SHARD, fun ?MODULE:do_clear/1, [ + fun( + #retained_message{expiry_time = ExpiryTime} + ) -> + (ExpiryTime =/= 0) and (ExpiryTime < NowMs) + end + ]), ok. -do_clear_expired() -> - NowMs = erlang:system_time(millisecond), +-spec do_clear(fun((retained_message()) -> boolean())) -> ok. +do_clear(Pred) -> QH = qlc:q([ TopicTokens - || #retained_message{ - topic = TopicTokens, - expiry_time = ExpiryTime - } <- mnesia:table(?TAB_MESSAGE, [{lock, write}]), - (ExpiryTime =/= 0) and (ExpiryTime < NowMs) + || #retained_message{topic = TopicTokens} = Data <- mnesia:table(?TAB_MESSAGE, [{lock, write}]), + Pred(Data) ]), QC = qlc:cursor(QH), clear_batch(db_indices(write), QC). @@ -263,6 +270,14 @@ clean(_) -> _ = mria:clear_table(?TAB_INDEX), ok. +clean_by_clientid(_, ClientId) -> + {atomic, _} = mria:transaction(?RETAINER_SHARD, fun ?MODULE:do_clear/1, [ + fun(Msg) -> + Msg#retained_message.msg#message.from =:= ClientId + end + ]), + ok. + size(_) -> table_size(). diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index 09e6c4bb4..23d4aee98 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -626,6 +626,66 @@ t_get_basic_usage_info(_Config) -> ?assertEqual(#{retained_messages => 5}, emqx_retainer:get_basic_usage_info()), ok. +t_banned_clean(_) -> + ClientId1 = <<"bc1">>, + ClientId2 = <<"bc2">>, + {ok, C1} = emqtt:start_link([{clientid, ClientId1}, {clean_start, true}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C1), + + {ok, C2} = emqtt:start_link([{clientid, ClientId2}, {clean_start, true}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C2), + + [ + begin + emqtt:publish( + Conn, + <<"bc/0/", ClientId/binary>>, + <<"this is a retained message 0">>, + [{qos, 0}, {retain, true}] + ), + emqtt:publish( + Conn, + <<"bc/1/", ClientId/binary>>, + <<"this is a retained message 1">>, + [{qos, 0}, {retain, true}] + ) + end + || {ClientId, Conn} <- lists:zip([ClientId1, ClientId2], [C1, C2]) + ], + + emqtt:publish( + C2, + <<"bc/2/", ClientId2/binary>>, + <<"this is a retained message 2">>, + [{qos, 0}, {retain, true}] + ), + + timer:sleep(500), + {ok, List} = emqx_retainer:page_read(<<"bc/+/+">>, 1, 10), + ?assertEqual(5, length(List)), + + Now = erlang:system_time(second), + Who = {clientid, ClientId2}, + emqx_banned:create(#{ + who => Who, + by => <<"test">>, + reason => <<"test">>, + at => Now, + until => Now + 120, + clean => true + }), + + timer:sleep(500), + + {ok, List2} = emqx_retainer:page_read(<<"bc/#">>, 1, 10), + ?assertEqual(2, length(List2)), + + emqx_banned:delete(Who), + emqx_retainer:clean(), + timer:sleep(500), + ok = emqtt:disconnect(C1), + ok = emqtt:disconnect(C2). + %% test whether the app can start normally after disabling emqx_retainer %% fix: https://github.com/emqx/emqx/pull/8911 test_disable_then_start(_Config) -> From b2eed47976dd60b008cec6107437dc4b6a53be74 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 18 Oct 2022 13:36:08 +0800 Subject: [PATCH 055/165] chore: bump retainer version && update CHANGES --- CHANGES-5.0.md | 1 + apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 0f617678d..6e78d3532 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -9,6 +9,7 @@ Prior to this fix, the message publish APIs (`api/v5/publish` and `api/v5/publish/bulk`) echos the message back to the client in HTTP body. This change fixed it to only send back the message ID. * Add /trace/:name/log_detail HTTP API to return trace file's size and mtime [#9152](https://github.com/emqx/emqx/pull/9152) +* Allow clear retained/delayed data when client is banned.[#9139](https://github.com/emqx/emqx/pull/9139) ## Bug fixes diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 888335ab4..c49794cfe 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.5"}, + {vsn, "5.0.6"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx]}, From 3f1f2443108c7ceb4e9bcf29d6416fa7d141454f Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 12 Oct 2022 19:27:57 +0800 Subject: [PATCH 056/165] fix: fix bpapi versions --- apps/emqx/priv/bpapi.versions | 1 + apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index f5c515cce..da3e5d023 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -8,6 +8,7 @@ {emqx_conf,1}. {emqx_conf,2}. {emqx_dashboard,1}. +{emqx_delayed,1}. {emqx_delayed,2}. {emqx_exhook,1}. {emqx_gateway_api_listeners,1}. diff --git a/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl b/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl index bb09e47eb..e08ab785a 100644 --- a/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl +++ b/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl @@ -28,7 +28,7 @@ -include_lib("emqx/include/bpapi.hrl"). introduced_in() -> - "5.0.0". + "5.0.9". -spec get_delayed_message(node(), binary()) -> emqx_delayed:with_id_return(map()) | emqx_rpc:badrpc(). From bdaadda479ce27b640e4fe3238e12cca7dea5ece Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 18 Oct 2022 11:34:43 +0800 Subject: [PATCH 057/165] fix(banned): fix nit code --- .../src/emqx_mgmt_api_banned.erl | 1 + apps/emqx_modules/src/emqx_delayed.erl | 3 +- .../src/proto/emqx_delayed_proto_v2.erl | 10 ++++--- apps/emqx_modules/test/emqx_delayed_SUITE.erl | 28 ++++++++++--------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index e3a99038c..1eacc67c2 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -155,6 +155,7 @@ fields(ban) -> hoconsc:mk(boolean(), #{ desc => ?DESC(clean), required => false, + default => false, example => false })} ]. diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index f703c9dce..6a8b8de5b 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -263,8 +263,7 @@ post_config_update(_KeyPath, _ConfigReq, NewConf, _OldConf, _AppEnvs) -> clean_by_clientid(ClientId) -> Nodes = mria_mnesia:running_nodes(), - _ = [emqx_delayed_proto_v2:clean_by_clientid(Node, ClientId) || Node <- Nodes], - ok. + emqx_delayed_proto_v2:clean_by_clientid(Nodes, ClientId). do_clean_by_clientid(ClientId) -> ets:select_delete( diff --git a/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl b/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl index e08ab785a..866b2cd40 100644 --- a/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl +++ b/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl @@ -27,6 +27,8 @@ -include_lib("emqx/include/bpapi.hrl"). +-define(TIMEOUT, 15000). + introduced_in() -> "5.0.9". @@ -39,7 +41,7 @@ get_delayed_message(Node, Id) -> delete_delayed_message(Node, Id) -> rpc:call(Node, emqx_delayed, delete_delayed_message, [Id]). --spec clean_by_clientid(node(), emqx_types:clientid()) -> - emqx_delayed:with_id_return() | emqx_rpc:badrpc(). -clean_by_clientid(Node, ClientID) -> - rpc:call(Node, emqx_delayed, do_clean_by_clientid, [ClientID]). +-spec clean_by_clientid(list(node()), emqx_types:clientid()) -> + emqx_rpc:erpc_multicall(). +clean_by_clientid(Nodes, ClientID) -> + erpc:multicall(Nodes, emqx_delayed, do_clean_by_clientid, [ClientID], ?TIMEOUT). diff --git a/apps/emqx_modules/test/emqx_delayed_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_SUITE.erl index f19d9388c..bd1de3849 100644 --- a/apps/emqx_modules/test/emqx_delayed_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_SUITE.erl @@ -252,21 +252,23 @@ t_banned_clean(_) -> Now = erlang:system_time(second), Who = {clientid, ClientId2}, - emqx_banned:create(#{ - who => Who, - by => <<"test">>, - reason => <<"test">>, - at => Now, - until => Now + 120, - clean => true - }), + try + emqx_banned:create(#{ + who => Who, + by => <<"test">>, + reason => <<"test">>, + at => Now, + until => Now + 120, + clean => true + }), - timer:sleep(500), + timer:sleep(500), - ?assertMatch(#{meta := #{count := 2}}, emqx_delayed:list(#{page => 1, limit => 10})), - - emqx_banned:delete(Who), - emqx_delayed:clean_by_clientid(ClientId1), + ?assertMatch(#{meta := #{count := 2}}, emqx_delayed:list(#{page => 1, limit => 10})) + after + emqx_banned:delete(Who), + emqx_delayed:clean_by_clientid(ClientId1) + end, timer:sleep(500), ok = emqtt:disconnect(C1), ok = emqtt:disconnect(C2). From 7b3c67fbe9faab5c962809f989625be5edab7adf Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 18 Oct 2022 13:47:57 +0800 Subject: [PATCH 058/165] test(shared_sub): ensure snabbkaffe will be stopped --- apps/emqx/test/emqx_shared_sub_SUITE.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index f048df397..45d9b4fad 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -821,6 +821,7 @@ t_dispatch_qos0(Config) when is_list(Config) -> t_redispatch_when_kicked({init, Config}) when is_list(Config) -> Config; t_redispatch_when_kicked({'end', Config}) when is_list(Config) -> + snabbkaffe:stop(), ok; t_redispatch_when_kicked(_) -> ok = ensure_config(sticky, true), From cd40acb0357faa0e053bb7f129057cb7130eb439 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 20 Oct 2022 08:54:29 +0200 Subject: [PATCH 059/165] chore: fix ssl.enable variable in enterprise helm chart --- deploy/charts/emqx-enterprise/templates/certificate.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/charts/emqx-enterprise/templates/certificate.yaml b/deploy/charts/emqx-enterprise/templates/certificate.yaml index 36b7f6521..9a2ed969a 100644 --- a/deploy/charts/emqx-enterprise/templates/certificate.yaml +++ b/deploy/charts/emqx-enterprise/templates/certificate.yaml @@ -1,4 +1,4 @@ -{{- if and (.Values.ssl.enable) (not .Values.ssl.useExisting) -}} +{{- if and (.Values.ssl.enabled) (not .Values.ssl.useExisting) -}} --- apiVersion: cert-manager.io/v1 kind: Certificate From c7a90d73b25741be9b616ca325277ba05ccc78fb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 19 Oct 2022 17:09:37 +0200 Subject: [PATCH 060/165] fix(shared): do not redispatch shared messages for certain shutdown For takeover, there should be no message re-dispatch because the messages will be retried by the new session. For kick, messages should not be re-dispatched for security reason. i.e. if admin has identified that there are malicious messages stored in persisted sessions, killing the session should not cause messages to be re-dispatched --- apps/emqx/src/emqx_session.erl | 10 +- apps/emqx/test/emqx_shared_sub_SUITE.erl | 135 ++++++++++++++++++++++- 2 files changed, 137 insertions(+), 8 deletions(-) diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index b285d0a88..51e57462b 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -802,8 +802,7 @@ replay(ClientInfo, Session = #session{inflight = Inflight}) -> -spec terminate(emqx_types:clientinfo(), Reason :: term(), session()) -> ok. terminate(ClientInfo, Reason, Session) -> run_terminate_hooks(ClientInfo, Reason, Session), - Reason =/= takenover andalso - redispatch_shared_messages(Session), + maybe_redispatch_shared_messages(Reason, Session), ok. run_terminate_hooks(ClientInfo, discarded, Session) -> @@ -813,6 +812,13 @@ run_terminate_hooks(ClientInfo, takenover, Session) -> run_terminate_hooks(ClientInfo, Reason, Session) -> run_hook('session.terminated', [ClientInfo, Reason, info(Session)]). +maybe_redispatch_shared_messages(takenover, _Session) -> + ok; +maybe_redispatch_shared_messages(kicked, _Session) -> + ok; +maybe_redispatch_shared_messages(_Reason, Session) -> + redispatch_shared_messages(Session). + redispatch_shared_messages(#session{inflight = Inflight, mqueue = Q}) -> AllInflights = emqx_inflight:to_list(fun sort_fun/2, Inflight), F = fun diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 291286aa2..dab4c698a 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -750,11 +750,16 @@ t_dispatch_qos2(Config) when is_list(Config) -> ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message3)), ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message4)), + %% assert client 2 receives two messages, they are eiter 1,3 or 2,4 depending + %% on if it's picked as the first one for round_robin MsgRec1 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P1}}, P1), MsgRec2 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P2}}, P2), - %% assert hello2 > hello1 or hello4 > hello3 - ?assert(MsgRec2 > MsgRec1), - + case MsgRec2 of + <<"hello3">> -> + ?assertEqual(<<"hello1">>, MsgRec1); + <<"hello4">> -> + ?assertEqual(<<"hello2">>, MsgRec1) + end, sys:resume(ConnPid1), %% emqtt subscriber automatically sends PUBREC, but since auto_ack is set to false %% so it will never send PUBCOMP, hence EMQX should not attempt to send @@ -767,8 +772,14 @@ t_dispatch_qos2(Config) when is_list(Config) -> kill_process(ConnPid1), %% client 2 should receive the message MsgRec4 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P4}}, P4), - %% assert hello2 > hello1 or hello4 > hello3 - ?assert(MsgRec4 > MsgRec3), + case MsgRec2 of + <<"hello3">> -> + ?assertEqual(<<"hello2">>, MsgRec3), + ?assertEqual(<<"hello4">>, MsgRec4); + <<"hello4">> -> + ?assertEqual(<<"hello1">>, MsgRec3), + ?assertEqual(<<"hello3">>, MsgRec4) + end, emqtt:stop(ConnPid2), ok. @@ -817,17 +828,129 @@ t_dispatch_qos0(Config) when is_list(Config) -> emqtt:stop(ConnPid2), ok. +t_session_takeover({init, Config}) when is_list(Config) -> + Config; +t_session_takeover({'end', Config}) when is_list(Config) -> + ok; +t_session_takeover(Config) when is_list(Config) -> + Topic = <<"t1/a">>, + ClientId = iolist_to_binary("c" ++ integer_to_list(erlang:system_time())), + Opts = [ + {clientid, ClientId}, + {auto_ack, true}, + {proto_ver, v5}, + {clean_start, false}, + {properties, #{'Session-Expiry-Interval' => 60}} + ], + {ok, ConnPid1} = emqtt:start_link(Opts), + %% with the same client ID, start another client + {ok, ConnPid2} = emqtt:start_link(Opts), + {ok, _} = emqtt:connect(ConnPid1), + emqtt:subscribe(ConnPid1, {<<"$share/t1/", Topic/binary>>, _QoS = 1}), + Message1 = emqx_message:make(<<"dummypub">>, 2, Topic, <<"hello1">>), + Message2 = emqx_message:make(<<"dummypub">>, 2, Topic, <<"hello2">>), + Message3 = emqx_message:make(<<"dummypub">>, 2, Topic, <<"hello3">>), + Message4 = emqx_message:make(<<"dummypub">>, 2, Topic, <<"hello4">>), + %% Make sure client1 is functioning + ?assertMatch([_], emqx:publish(Message1)), + {true, _} = last_message(<<"hello1">>, [ConnPid1]), + %% Kill client1 + emqtt:stop(ConnPid1), + %% publish another message (should end up in client1's session) + ?assertMatch([_], emqx:publish(Message2)), + %% connect client2 (with the same clientid) + + %% should trigger session take over + {ok, _} = emqtt:connect(ConnPid2), + ?assertMatch([_], emqx:publish(Message3)), + ?assertMatch([_], emqx:publish(Message4)), + {true, _} = last_message(<<"hello2">>, [ConnPid1]), + {true, _} = last_message(<<"hello3">>, [ConnPid1]), + {true, _} = last_message(<<"hello4">>, [ConnPid1]), + ?assertEqual([], collect_msgs(timer:seconds(2))), + emqtt:stop(ConnPid2), + ok. + +t_session_kicked({init, Config}) when is_list(Config) -> + emqx_config:put_zone_conf(default, [mqtt, max_inflight], 1), + Config; +t_session_kicked({'end', Config}) when is_list(Config) -> + emqx_config:put_zone_conf(default, [mqtt, max_inflight], 0); +t_session_kicked(Config) when is_list(Config) -> + ok = ensure_config(round_robin, _AckEnabled = false), + Topic = <<"foo/bar/1">>, + ClientId1 = <<"ClientId1">>, + ClientId2 = <<"ClientId2">>, + + {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}, {auto_ack, false}]), + {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}, {auto_ack, true}]), + {ok, _} = emqtt:connect(ConnPid1), + {ok, _} = emqtt:connect(ConnPid2), + + emqtt:subscribe(ConnPid1, {<<"$share/group/foo/bar/#">>, 2}), + emqtt:subscribe(ConnPid2, {<<"$share/group/foo/bar/#">>, 2}), + + Message1 = emqx_message:make(ClientId1, 2, Topic, <<"hello1">>), + Message2 = emqx_message:make(ClientId1, 2, Topic, <<"hello2">>), + Message3 = emqx_message:make(ClientId1, 2, Topic, <<"hello3">>), + Message4 = emqx_message:make(ClientId1, 2, Topic, <<"hello4">>), + ct:sleep(100), + + ok = sys:suspend(ConnPid1), + + %% One message is inflight + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message1)), + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message2)), + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message3)), + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message4)), + + %% assert client 2 receives two messages, they are eiter 1,3 or 2,4 depending + %% on if it's picked as the first one for round_robin + MsgRec1 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P1}}, P1), + MsgRec2 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P2}}, P2), + case MsgRec2 of + <<"hello3">> -> + ?assertEqual(<<"hello1">>, MsgRec1); + <<"hello4">> -> + ?assertEqual(<<"hello2">>, MsgRec1) + end, + sys:resume(ConnPid1), + %% emqtt subscriber automatically sends PUBREC, but since auto_ack is set to false + %% so it will never send PUBCOMP, hence EMQX should not attempt to send + %% the 4th message yet since max_inflight is 1. + MsgRec3 = ?WAIT(2000, {publish, #{client_pid := ConnPid1, payload := P3}}, P3), + case MsgRec2 of + <<"hello3">> -> + ?assertEqual(<<"hello2">>, MsgRec3); + <<"hello4">> -> + ?assertEqual(<<"hello1">>, MsgRec3) + end, + %% no message expected + ?assertEqual([], collect_msgs(0)), + %% now kick client 1 + kill_process(ConnPid1, fun(_Pid) -> emqx_cm:kick_session(ClientId1) end), + %% client 2 should NOT receive the message + ?assertEqual([], collect_msgs(1000)), + emqtt:stop(ConnPid2), + ?assertEqual([], collect_msgs(0)), + ok. + %%-------------------------------------------------------------------- %% help functions %%-------------------------------------------------------------------- kill_process(Pid) -> + kill_process(Pid, fun(_) -> erlang:exit(Pid, kill) end). + +kill_process(Pid, WithFun) -> _ = unlink(Pid), _ = monitor(process, Pid), - erlang:exit(Pid, kill), + _ = WithFun(Pid), receive {'DOWN', _, process, Pid, _} -> ok + after 10_000 -> + error(timeout) end. collect_msgs(Timeout) -> From fa020d036f702a0b702dee8b6ba45a49b8ed62b1 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 20 Oct 2022 15:30:52 +0800 Subject: [PATCH 061/165] chore(docs): translate Max Awaiting PUBREL confs --- apps/emqx/i18n/emqx_schema_i18n.conf | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 17e59a4dd..ae9da4579 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -869,23 +869,25 @@ When set to true, invalid utf8 strings in for example client ID, topic name, etc mqtt_max_awaiting_rel { desc { - en: """Maximum QoS 2 packets (Client -> Broker) awaiting PUBREL.""" - zh: """PUBREL (Client -> Broker) 最大等待队列长度。""" + en: """EMQX creates a wait queue in the client's session to accept QoS 2 messages from the client until this QoS 2 message transmission is complete or is discarded due to timeout.
+After this queue is full, the processing of QoS 2 messages will be stopped.""" + zh: """EMQX 会在客户端的会话中创建一个等待队列来接受客户端的 QoS 2 消息,直到这条 QoS 2 消息传输完成或由于超时被丢弃。
+该队列满后会停止处理 QoS 2 消息。""" } label: { en: """Max Awaiting PUBREL""" - zh: """Max Awaiting PUBREL""" + zh: """PUBREL 等待队列长度""" } } mqtt_await_rel_timeout { desc { - en: """The QoS 2 messages (Client -> Broker) will be dropped if awaiting PUBREL timeout.""" - zh: """PUBREL (Client -> Broker) 最大等待时间,超时则会被丢弃。""" + en: """The QoS 2 messages (Client -> Broker) handle flow will be ignored if awaiting PUBREL timeout. This QoS 2 message can be published successfully.""" + zh: """如果等待 PUBREL 超时,QoS 2消息(Client -> Broker)处理流程将被忽略。这条 QoS 2 仍然会成功发布。""" } label: { en: """Max Awaiting PUBREL TIMEOUT""" - zh: """Max Awaiting PUBREL TIMEOUT""" + zh: """PUBREL 最大等待时间""" } } From d0120e1edcfa5594bd9b76aa3cecbc216cca7333 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 20 Oct 2022 09:51:32 +0200 Subject: [PATCH 062/165] chore(gen_rpc): Bump version to 3.0.0 --- CHANGES-5.0.md | 1 + apps/emqx_conf/i18n/emqx_conf_schema.conf | 11 +++++++++++ apps/emqx_conf/src/emqx_conf_schema.erl | 9 +++++++++ 3 files changed, 21 insertions(+) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 6e78d3532..f885aae8f 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -10,6 +10,7 @@ This change fixed it to only send back the message ID. * Add /trace/:name/log_detail HTTP API to return trace file's size and mtime [#9152](https://github.com/emqx/emqx/pull/9152) * Allow clear retained/delayed data when client is banned.[#9139](https://github.com/emqx/emqx/pull/9139) +* Update `gen_rpc` library to version 3.0 [#9187](https://github.com/emqx/emqx/pull/9187) ## Bug fixes diff --git a/apps/emqx_conf/i18n/emqx_conf_schema.conf b/apps/emqx_conf/i18n/emqx_conf_schema.conf index fb986785b..6ba57f9ee 100644 --- a/apps/emqx_conf/i18n/emqx_conf_schema.conf +++ b/apps/emqx_conf/i18n/emqx_conf_schema.conf @@ -973,6 +973,17 @@ until the RPC connection is considered lost.""" } } + rpc_insecure_fallback { + desc { + en: """Enable compatibility with old RPC authentication.""" + zh: """启用与旧RPC认证的兼容性。""" + } + label { + en: "RPC insecure fallback" + zh: "RPC不安全的后退" + } + } + log_file_handlers { desc { en: """File-based log handlers.""" diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index ec61cb508..4de952d2e 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -814,6 +814,15 @@ fields("rpc") -> default => "1MB", desc => ?DESC(rpc_socket_buffer) } + )}, + {"insecure_fallback", + sc( + boolean(), + #{ + mapping => "gen_rpc.insecure_auth_fallback_allowed", + default => true, + desc => ?DESC(rpc_insecure_fallback) + } )} ]; fields("log") -> From b3ffe89f3521f93207de36765a089e0eda264f39 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 19 Oct 2022 18:16:38 +0200 Subject: [PATCH 063/165] test: assert message receive pid is in the expected pids list --- apps/emqx/test/emqx_shared_sub_SUITE.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index dab4c698a..bc519117b 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -465,7 +465,7 @@ last_message(ExpectedPayload, Pids) -> last_message(ExpectedPayload, Pids, Timeout) -> receive {publish, #{client_pid := Pid, payload := ExpectedPayload}} -> - ct:pal("~p ====== ~p", [Pids, Pid]), + ?assert(lists:member(Pid, Pids)), {true, Pid} after Timeout -> ct:pal("not yet"), @@ -864,9 +864,9 @@ t_session_takeover(Config) when is_list(Config) -> {ok, _} = emqtt:connect(ConnPid2), ?assertMatch([_], emqx:publish(Message3)), ?assertMatch([_], emqx:publish(Message4)), - {true, _} = last_message(<<"hello2">>, [ConnPid1]), - {true, _} = last_message(<<"hello3">>, [ConnPid1]), - {true, _} = last_message(<<"hello4">>, [ConnPid1]), + {true, _} = last_message(<<"hello2">>, [ConnPid2]), + {true, _} = last_message(<<"hello3">>, [ConnPid2]), + {true, _} = last_message(<<"hello4">>, [ConnPid2]), ?assertEqual([], collect_msgs(timer:seconds(2))), emqtt:stop(ConnPid2), ok. From 67dd733e8d3ede357d109d0068725d1eeeeb451f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 20 Oct 2022 18:34:35 +0200 Subject: [PATCH 064/165] chore: bump to gun 1.3.9 1.3.7 had a bug which caused ssl_passive message unhandled --- mix.exs | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index a2863f137..43024c116 100644 --- a/mix.exs +++ b/mix.exs @@ -71,7 +71,7 @@ defmodule EMQXUmbrella.MixProject do {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, # in conflict by ehttpc and emqtt - {:gun, github: "emqx/gun", tag: "1.3.7", override: true}, + {:gun, github: "emqx/gun", tag: "1.3.9", override: true}, # in conflict by emqx_connectior and system_monitor {:epgsql, github: "emqx/epgsql", tag: "4.7-emqx.2", override: true}, # in conflict by mongodb and eredis_cluster diff --git a/rebar.config b/rebar.config index 9be69d963..f415b5901 100644 --- a/rebar.config +++ b/rebar.config @@ -48,7 +48,7 @@ , {redbug, "2.0.7"} , {gpb, "4.19.5"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} - , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.7"}}} + , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}} , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.3.0"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} From 05793ef1e516400694c12cd96a8a826324ab235d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 20 Oct 2022 19:13:09 +0200 Subject: [PATCH 065/165] chore: upgrade to dashboard v1.1.0 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 74204afac..eeabaed33 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.0.9 +export EMQX_DASHBOARD_VERSION ?= v1.1.0 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.0 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 From d1332b72e7f7e4075bfb7aa575449560baa9a146 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Oct 2022 21:33:29 +0200 Subject: [PATCH 066/165] feat(api/publish): return detailed publish results Prior to this change, the publish API returns 200 in most of the cases. This change provides more insights to the publish result. For single message publish endpoint (`publish/`): HTTP error codes are: 200: Everything is OK 202: No subscriber for the topic 400: When mesage is invalid message. e.g. bad topic name or QoS out of range. 503: Failed to dispatch the message. e.g. during EMQX restart. The response body is a JSON object with two fields `message_id`, and `publish_result`. The `message_id` is a globally unique ID for tracing. `publish_result` is `"OK"` when the message is delivered to at least one subscriber. Otherwise `"no_subscriber"`. `publish_result` may also be some other informative message to hint the failure result, the content of which may change in the future. For `publish/bulk` endpoint: 200: When all message in the bulk are published OK 202: If at least one message in the bulk had `"no_subscriber"` result 400: When mesage is invalid message. e.g. bad topic name or QoS out of range. 503: When there is at least one message failed at dispatch. The reply body is a list of JSON objects having the same layout as for hte `publish` endpoint. --- apps/emqx/src/emqx_message.erl | 15 +- .../i18n/emqx_mgmt_api_publish_i18n.conf | 127 +++++++++ .../src/emqx_mgmt_api_publish.erl | 241 ++++++++++++++--- .../test/emqx_mgmt_api_publish_SUITE.erl | 246 +++++++++++++++++- .../test/emqx_mgmt_api_test_util.erl | 18 +- 5 files changed, 604 insertions(+), 43 deletions(-) create mode 100644 apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf diff --git a/apps/emqx/src/emqx_message.erl b/apps/emqx/src/emqx_message.erl index ae74a614b..03f7ca6a2 100644 --- a/apps/emqx/src/emqx_message.erl +++ b/apps/emqx/src/emqx_message.erl @@ -74,7 +74,8 @@ to_map/1, to_log_map/1, to_list/1, - from_map/1 + from_map/1, + estimate_size/1 ]). -export_type([message_map/0]). @@ -175,6 +176,18 @@ make(MsgId, From, QoS, Topic, Payload, Flags, Headers) when timestamp = Now }. +%% optimistic esitmation of a message size after serialization +%% not including MQTT v5 message headers/user properties etc. +-spec estimate_size(emqx_types:message()) -> non_neg_integer(). +estimate_size(#message{topic = Topic, payload = Payload}) -> + FixedHeaderSize = 1, + VarLenSize = 4, + TopicSize = iolist_size(Topic), + PayloadSize = iolist_size(Payload), + PacketIdSize = 2, + TopicLengthSize = 2, + FixedHeaderSize + VarLenSize + TopicLengthSize + TopicSize + PacketIdSize + PayloadSize. + -spec id(emqx_types:message()) -> maybe(binary()). id(#message{id = Id}) -> Id. diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf new file mode 100644 index 000000000..2a7c9def8 --- /dev/null +++ b/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf @@ -0,0 +1,127 @@ + +emqx_mgmt_api_publish { + publish_api { + desc { + en: """ +Publish one message.
+Possible HTTP status response codes are:
+200: The message is delivered to at least one subscriber;
+202: No matched subscribers;
+400: Message is invalid. for example bad topic name, or QoS is out of range;
+503: Failed to deliver the message to subscriber(s);
+""" + zh: """ +发布一个消息。
+可能的 HTTP 状态码如下:
+200: 消息被成功发送到至少一个订阅。
+202: 没有匹配到任何订阅。
+400: 消息编码错误,如非法主题,或 QoS 超出范围等。
+503: 服务重启等过程中导致转发失败。

+""" + } + } + publish_bulk_api { + desc { + en: """ +Publish a batch of messages.
+Possible HTTP response status code are:
+200: All messages are delivered to at least one subscriber;
+202: At least one message was not delivered to any subscriber;
+400: At least one message is invalid. For example bad topic name, or QoS is out of range;
+503: Failed to deliver at least one of the messages;
+ +In case there is at lest one invalid message in the batch, the HTTP response body +is the same as for /publish API.
+Otherwise the HTTP response body is an array of JSON objects indicating the publish +result of each individual message in the batch. +""" + zh: """ +批量发布一组消息。
+可能的 HTTP 状态码如下:
+200: 所有的消息都被成功发送到至少一个订阅。
+202: 至少有一个消息没有匹配到任何订阅。
+400: 至少有一个消息编码错误,如非法主题,或 QoS 超出范围等。
+503: 至少有一个小因为服务重启的原因导致转发失败。
+ +请求的 Body 或者 Body 中包含的某个消息无法通过 API 规范的类型检查时,HTTP 响应的消息与发布单个消息的 API + /publish 是一样的。 +如果所有的消息都是合法的,那么 HTTP 返回的内容是一个 JSON 数组,每个元素代表了该消息转发的状态。 + +""" + } + } + + topic_name { + desc { + en: "Topic Name" + zh: "主题名称" + } + } + qos { + desc { + en: "MQTT message QoS" + zh: "MQTT 消息的 QoS" + } + } + clientid { + desc { + en: "Each message can be published as if it is done on behalf of an MQTT client whos ID can be specified in this field." + zh: "每个消息都可以带上一个 MQTT 客户端 ID,用于模拟 MQTT 客户端的发布行为。" + } + } + payload { + desc { + en: "The MQTT message payload." + zh: "MQTT 消息体。" + } + } + retain { + desc { + en: "A boolean field to indicate if this message should be retained." + zh: "布尔型字段,用于表示该消息是否保留消息。" + } + } + payload_encoding { + desc { + en: "MQTT Payload Encoding, base64 or plain. When set to base64, the message is decoded before it is published." + zh: "MQTT 消息体的编码方式,可以是 base64plain。当设置为 base64 时,消息在发布前会先被解码。" + } + } + message_id { + desc { + en: "A globally unique message ID for correlation/tracing." + zh: "全局唯一的一个消息 ID,方便用于关联和追踪。" + } + } + reason_code { + desc { + en: """ +The MQTT reason code, as the same ones used in PUBACK packet.
+Currently supported codes are:
+ +16(0x10): No matching subscribers;
+131(0x81): Error happened when dispatching the message. e.g. during EMQX restart;
+144(0x90): Topic name invalid;
+151(0x97): Publish rate limited, or message size exceeded limit. The global size limit can be configured with mqtt.max_packet_size
+NOTE: The message size is estimated with the received topic and payload size, meaning the actual size of serialized bytes (when sent to MQTT subscriber) +might be slightly over the limit. +""" + zh: """ +MQTT 消息发布的错误码,这些错误码也是 MQTT 规范中 PUBACK 消息可能携带的错误码。
+当前支持如下错误码:
+ +16(0x10):没能匹配到任何订阅;
+131(0x81):消息转发时发生错误,例如 EMQX 服务重启;
+144(0x90):主题名称非法;
+151(0x97):受到了速率限制,或者消息尺寸过大。全局消息大小限制可以通过配置项 mqtt.max_packet_size 来进行修改。
+注意:消息尺寸的是通过主题和消息体的字节数进行估算的。具体发布时所占用的字节数可能会稍大于这个估算的值。 +""" + } + } + error_message { + desc { + en: "Describes the failure reason in detail." + zh: "失败的详细原因。" + } + } +} diff --git a/apps/emqx_management/src/emqx_mgmt_api_publish.erl b/apps/emqx_management/src/emqx_mgmt_api_publish.erl index 0e05a3875..1678c56e0 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_publish.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_publish.erl @@ -16,7 +16,15 @@ -module(emqx_mgmt_api_publish). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("typerefl/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-define(ALL_IS_WELL, 200). +-define(PARTIALLY_OK, 202). +-define(BAD_REQUEST, 400). +-define(DISPATCH_ERROR, 503). -behaviour(minirest_api). @@ -42,11 +50,14 @@ schema("/publish") -> #{ 'operationId' => publish, post => #{ - description => <<"Publish Message">>, + description => ?DESC(publish_api), tags => [<<"Publish">>], 'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, publish_message)), responses => #{ - 200 => hoconsc:mk(hoconsc:ref(?MODULE, publish_message_info)) + ?ALL_IS_WELL => hoconsc:mk(hoconsc:ref(?MODULE, publish_ok)), + ?PARTIALLY_OK => hoconsc:mk(hoconsc:ref(?MODULE, publish_error)), + ?BAD_REQUEST => bad_request_schema(), + ?DISPATCH_ERROR => hoconsc:mk(hoconsc:ref(?MODULE, publish_error)) } } }; @@ -54,44 +65,58 @@ schema("/publish/bulk") -> #{ 'operationId' => publish_batch, post => #{ - description => <<"Publish Messages">>, + description => ?DESC(publish_bulk_api), tags => [<<"Publish">>], 'requestBody' => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, publish_message)), #{}), responses => #{ - 200 => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, publish_message_info)), #{}) + ?ALL_IS_WELL => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, publish_ok)), #{}), + ?PARTIALLY_OK => hoconsc:mk( + hoconsc:array(hoconsc:ref(?MODULE, publish_error)), #{} + ), + ?BAD_REQUEST => bad_request_schema(), + ?DISPATCH_ERROR => hoconsc:mk( + hoconsc:array(hoconsc:ref(?MODULE, publish_error)), #{} + ) } } }. +bad_request_schema() -> + Union = hoconsc:union([ + hoconsc:ref(?MODULE, bad_request), + hoconsc:array(hoconsc:ref(?MODULE, publish_error)) + ]), + hoconsc:mk(Union, #{}). + fields(message) -> [ {topic, hoconsc:mk(binary(), #{ - desc => <<"Topic Name">>, + desc => ?DESC(topic_name), required => true, example => <<"api/example/topic">> })}, {qos, hoconsc:mk(emqx_schema:qos(), #{ - desc => <<"MQTT QoS">>, + desc => ?DESC(qos), required => false, default => 0 })}, {clientid, hoconsc:mk(binary(), #{ - desc => <<"From client ID">>, + desc => ?DESC(clientid), required => false, example => <<"api_example_client">> })}, {payload, hoconsc:mk(binary(), #{ - desc => <<"MQTT Payload">>, + desc => ?DESC(payload), required => true, example => <<"hello emqx api">> })}, {retain, hoconsc:mk(boolean(), #{ - desc => <<"MQTT Retain Message">>, + desc => ?DESC(retain), required => false, default => false })} @@ -100,53 +125,196 @@ fields(publish_message) -> [ {payload_encoding, hoconsc:mk(hoconsc:enum([plain, base64]), #{ - desc => <<"MQTT Payload Encoding, base64 or plain">>, + desc => ?DESC(payload_encoding), required => false, default => plain })} ] ++ fields(message); -fields(publish_message_info) -> +fields(publish_ok) -> [ {id, hoconsc:mk(binary(), #{ - desc => <<"A globally unique message ID">> + desc => ?DESC(message_id) + })} + ]; +fields(publish_error) -> + [ + {reason_code, + hoconsc:mk(integer(), #{ + desc => ?DESC(reason_code), + example => 16 + })}, + {message, + hoconsc:mk(binary(), #{ + desc => ?DESC(error_message), + example => <<"no_matching_subscribers">> + })} + ]; +fields(bad_request) -> + [ + {code, + hoconsc:mk(string(), #{ + desc => <<"BAD_REQUEST">> + })}, + {message, + hoconsc:mk(binary(), #{ + desc => ?DESC(error_message) })} ]. publish(post, #{body := Body}) -> case message(Body) of {ok, Message} -> - _ = emqx_mgmt:publish(Message), - {200, format_message_response(Message)}; - {error, R} -> - {400, 'BAD_REQUEST', to_binary(R)} + Res = emqx_mgmt:publish(Message), + publish_result_to_http_reply(Message, Res); + {error, Reason} -> + {?BAD_REQUEST, make_bad_req_reply(Reason)} end. publish_batch(post, #{body := Body}) -> case messages(Body) of {ok, Messages} -> - _ = [emqx_mgmt:publish(Message) || Message <- Messages], - {200, format_message_response(Messages)}; - {error, R} -> - {400, 'BAD_REQUEST', to_binary(R)} + ResList = lists:map( + fun(Message) -> + Res = emqx_mgmt:publish(Message), + publish_result_to_http_reply(Message, Res) + end, + Messages + ), + publish_results_to_http_reply(ResList); + {error, Reason} -> + {?BAD_REQUEST, make_bad_req_reply(Reason)} end. +make_bad_req_reply(invalid_topic_name) -> + make_publish_error_response(?RC_TOPIC_NAME_INVALID); +make_bad_req_reply(packet_too_large) -> + %% 0x95 RC_PACKET_TOO_LARGE is not a PUBACK reason code + %% This is why we use RC_QUOTA_EXCEEDED instead + make_publish_error_response(?RC_QUOTA_EXCEEDED, packet_too_large); +make_bad_req_reply(Reason) -> + make_publish_error_response(?RC_IMPLEMENTATION_SPECIFIC_ERROR, to_binary(Reason)). + +-spec is_ok_deliver({_NodeOrShare, _MatchedTopic, emqx_types:deliver_result()}) -> boolean(). +is_ok_deliver({_NodeOrShare, _MatchedTopic, ok}) -> true; +is_ok_deliver({_NodeOrShare, _MatchedTopic, {ok, _}}) -> true; +is_ok_deliver({_NodeOrShare, _MatchedTopic, {error, _}}) -> false. + +%% @hidden Map MQTT publish result reason code to HTTP status code. +%% MQTT reason code | Description | HTTP status code +%% 0 Success 200 +%% 16 No matching subscribers 202 +%% 128 Unspecified error 406 +%% 131 Implementation specific error 406 +%% 144 Topic Name invalid 400 +%% 151 Quota exceeded 400 +%% +%% %%%%%% Below error codes are not implemented so far %%%% +%% +%% If HTTP request passes HTTP authentication, it is considered trusted. +%% In the future, we may choose to check ACL for the provided MQTT Client ID +%% 135 Not authorized 401 +%% +%% %%%%%% Below error codes are not applicable %%%%%%% +%% +%% No user specified packet ID, so there should be no packet ID error +%% 145 Packet identifier is in use 400 +%% +%% No preceding payload format indicator to compare against. +%% Content-Type check should be done at HTTP layer but not here. +%% 153 Payload format invalid 400 +publish_result_to_http_reply(_Message, []) -> + %% matched no subscriber + {?PARTIALLY_OK, make_publish_error_response(?RC_NO_MATCHING_SUBSCRIBERS)}; +publish_result_to_http_reply(Message, PublishResult) -> + case lists:any(fun is_ok_deliver/1, PublishResult) of + true -> + %% delivered to at least one subscriber + OkBody = make_publish_response(Message), + {?ALL_IS_WELL, OkBody}; + false -> + %% this is quite unusual, matched, but failed to deliver + %% if this happens, the publish result log can be helpful + %% to idnetify the reason why publish failed + %% e.g. during emqx restart + ReasonString = <<"failed_to_dispatch">>, + ErrorBody = make_publish_error_response( + ?RC_IMPLEMENTATION_SPECIFIC_ERROR, ReasonString + ), + ?SLOG(warning, #{ + msg => ReasonString, + message_id => emqx_message:id(Message), + results => PublishResult + }), + {?DISPATCH_ERROR, ErrorBody} + end. + +%% @hidden Reply batch publish result. +%% 200 if all published OK. +%% 202 if at least one message matched no subscribers. +%% 503 for temp errors duing EMQX restart +publish_results_to_http_reply([_ | _] = ResList) -> + {Codes0, BodyL} = lists:unzip(ResList), + Codes = lists:usort(Codes0), + HasFailure = lists:member(?DISPATCH_ERROR, Codes), + All200 = (Codes =:= [?ALL_IS_WELL]), + Code = + case All200 of + true -> + %% All OK + ?ALL_IS_WELL; + false when not HasFailure -> + %% Partially OK + ?PARTIALLY_OK; + false -> + %% At least one failed + ?DISPATCH_ERROR + end, + {Code, BodyL}. + message(Map) -> + try + make_message(Map) + catch + throw:Reason -> + {error, Reason} + end. + +make_message(Map) -> Encoding = maps:get(<<"payload_encoding">>, Map, plain), - case encode_payload(Encoding, maps:get(<<"payload">>, Map)) of + case decode_payload(Encoding, maps:get(<<"payload">>, Map)) of {ok, Payload} -> From = maps:get(<<"clientid">>, Map, http_api), QoS = maps:get(<<"qos">>, Map, 0), Topic = maps:get(<<"topic">>, Map), Retain = maps:get(<<"retain">>, Map, false), - {ok, emqx_message:make(From, QoS, Topic, Payload, #{retain => Retain}, #{})}; + try + _ = emqx_topic:validate(name, Topic) + catch + error:_Reason -> + throw(invalid_topic_name) + end, + Message = emqx_message:make(From, QoS, Topic, Payload, #{retain => Retain}, #{}), + Size = emqx_message:estimate_size(Message), + (Size > size_limit()) andalso throw(packet_too_large), + {ok, Message}; {error, R} -> {error, R} end. -encode_payload(plain, Payload) -> +%% get the global packet size limit since HTTP API does not belong to any zone. +size_limit() -> + try + emqx_config:get([mqtt, max_packet_size]) + catch + _:_ -> + %% leave 1000 bytes for topic name etc. + ?MAX_PACKET_SIZE + end. + +decode_payload(plain, Payload) -> {ok, Payload}; -encode_payload(base64, Payload) -> +decode_payload(base64, Payload) -> try {ok, base64:decode(Payload)} catch @@ -154,6 +322,8 @@ encode_payload(base64, Payload) -> {error, {decode_base64_payload_failed, Payload}} end. +messages([]) -> + {errror, <<"empty_batch">>}; messages(List) -> messages(List, []). @@ -167,10 +337,23 @@ messages([MessageMap | List], Res) -> {error, R} end. -format_message_response(Messages) when is_list(Messages) -> - [format_message_response(Message) || Message <- Messages]; -format_message_response(#message{id = ID}) -> - #{id => emqx_guid:to_hexstr(ID)}. +make_publish_response(#message{id = ID}) -> + #{ + id => emqx_guid:to_hexstr(ID) + }. +make_publish_error_response(ReasonCode) -> + make_publish_error_response(ReasonCode, emqx_reason_codes:name(ReasonCode)). + +make_publish_error_response(ReasonCode, Msg) -> + #{ + reason_code => ReasonCode, + message => to_binary(Msg) + }. + +to_binary(Atom) when is_atom(Atom) -> + atom_to_binary(Atom); +to_binary(Msg) when is_binary(Msg) -> + Msg; to_binary(Term) -> - list_to_binary(io_lib:format("~p", [Term])). + list_to_binary(io_lib:format("~0p", [Term])). diff --git a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl index b4b3aa902..0ebaf7195 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl @@ -19,6 +19,7 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -define(CLIENTID, <<"api_clientid">>). -define(USERNAME, <<"api_username">>). @@ -36,6 +37,16 @@ init_per_suite(Config) -> end_per_suite(_) -> emqx_mgmt_api_test_util:end_suite(). +init_per_testcase(Case, Config) -> + ?MODULE:Case({init, Config}). + +end_per_testcase(Case, Config) -> + ?MODULE:Case({'end', Config}). + +t_publish_api({init, Config}) -> + Config; +t_publish_api({'end', _Config}) -> + ok; t_publish_api(_) -> {ok, Client} = emqtt:start_link(#{ username => <<"api_username">>, clientid => <<"api_clientid">> @@ -48,11 +59,113 @@ t_publish_api(_) -> Auth = emqx_mgmt_api_test_util:auth_header_(), Body = #{topic => ?TOPIC1, payload => Payload}, {ok, Response} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body), - ResponseMap = emqx_json:decode(Response, [return_maps]), - ?assertEqual([<<"id">>], maps:keys(ResponseMap)), + ResponseMap = decode_json(Response), + ?assertEqual([<<"id">>], lists:sort(maps:keys(ResponseMap))), ?assertEqual(ok, receive_assert(?TOPIC1, 0, Payload)), - emqtt:disconnect(Client). + emqtt:stop(Client). +t_publish_no_subscriber({init, Config}) -> + Config; +t_publish_no_subscriber({'end', _Config}) -> + ok; +t_publish_no_subscriber(_) -> + Payload = <<"hello">>, + Path = emqx_mgmt_api_test_util:api_path(["publish"]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Body = #{topic => ?TOPIC1, payload => Payload}, + {ok, Response} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body), + ResponseMap = decode_json(Response), + ?assertEqual([<<"message">>, <<"reason_code">>], lists:sort(maps:keys(ResponseMap))), + ?assertMatch(#{<<"reason_code">> := ?RC_NO_MATCHING_SUBSCRIBERS}, ResponseMap), + ok. + +t_publish_bad_topic({init, Config}) -> + Config; +t_publish_bad_topic({'end', _Config}) -> + ok; +t_publish_bad_topic(_) -> + Payload = <<"hello">>, + Path = emqx_mgmt_api_test_util:api_path(["publish"]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Body = #{topic => <<"not/a+/valid/topic">>, payload => Payload}, + ?assertMatch( + {error, {_, 400, _}}, emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body) + ). + +t_publish_bad_base64({init, Config}) -> + Config; +t_publish_bad_base64({'end', _Config}) -> + ok; +t_publish_bad_base64(_) -> + %% not a valid base64 + Payload = <<"hello">>, + Path = emqx_mgmt_api_test_util:api_path(["publish"]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Body = #{ + topic => <<"not/a+/valid/topic">>, payload => Payload, payload_encoding => <<"base64">> + }, + ?assertMatch( + {error, {_, 400, _}}, emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body) + ). + +t_publish_too_large({init, Config}) -> + MaxPacketSize = 100, + meck:new(emqx_config, [no_link, passthrough, no_history]), + meck:expect(emqx_config, get, fun + ([mqtt, max_packet_size]) -> + MaxPacketSize; + (Other) -> + meck:passthrough(Other) + end), + [{max_packet_size, MaxPacketSize} | Config]; +t_publish_too_large({'end', _Config}) -> + meck:unload(emqx_config), + ok; +t_publish_too_large(Config) -> + MaxPacketSize = proplists:get_value(max_packet_size, Config), + Payload = lists:duplicate(MaxPacketSize, $0), + Path = emqx_mgmt_api_test_util:api_path(["publish"]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Body = #{topic => <<"random/topic">>, payload => Payload}, + {error, {Summary, _Headers, ResponseBody}} = + emqx_mgmt_api_test_util:request_api( + post, + Path, + "", + Auth, + Body, + #{return_body => true} + ), + ?assertMatch({_, 400, _}, Summary), + ?assertMatch( + #{ + <<"reason_code">> := ?RC_QUOTA_EXCEEDED, + <<"message">> := <<"packet_too_large">> + }, + decode_json(ResponseBody) + ), + ok. + +t_publish_bad_topic_bulk({init, Config}) -> + Config; +t_publish_bad_topic_bulk({'end', _Config}) -> + ok; +t_publish_bad_topic_bulk(_Config) -> + Payload = <<"hello">>, + Path = emqx_mgmt_api_test_util:api_path(["publish", "bulk"]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Body = [ + #{topic => <<"not/a+/valid/topic">>, payload => Payload}, + #{topic => <<"good/topic">>, payload => Payload} + ], + ?assertMatch( + {error, {_, 400, _}}, emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body) + ). + +t_publish_bulk_api({init, Config}) -> + Config; +t_publish_bulk_api({'end', _Config}) -> + ok; t_publish_bulk_api(_) -> {ok, Client} = emqtt:start_link(#{ username => <<"api_username">>, clientid => <<"api_clientid">> @@ -63,19 +176,135 @@ t_publish_bulk_api(_) -> Payload = <<"hello">>, Path = emqx_mgmt_api_test_util:api_path(["publish", "bulk"]), Auth = emqx_mgmt_api_test_util:auth_header_(), - Body = [#{topic => ?TOPIC1, payload => Payload}, #{topic => ?TOPIC2, payload => Payload}], + Body = [ + #{ + topic => ?TOPIC1, + payload => Payload, + payload_encoding => plain + }, + #{ + topic => ?TOPIC2, + payload => base64:encode(Payload), + payload_encoding => base64 + } + ], {ok, Response} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body), - ResponseList = emqx_json:decode(Response, [return_maps]), + ResponseList = decode_json(Response), ?assertEqual(2, erlang:length(ResponseList)), lists:foreach( fun(ResponseMap) -> - ?assertEqual([<<"id">>], maps:keys(ResponseMap)) + ?assertMatch( + [<<"id">>], lists:sort(maps:keys(ResponseMap)) + ) end, ResponseList ), ?assertEqual(ok, receive_assert(?TOPIC1, 0, Payload)), ?assertEqual(ok, receive_assert(?TOPIC2, 0, Payload)), - emqtt:disconnect(Client). + emqtt:stop(Client). + +t_publish_no_subscriber_bulk({init, Config}) -> + Config; +t_publish_no_subscriber_bulk({'end', _Config}) -> + ok; +t_publish_no_subscriber_bulk(_) -> + {ok, Client} = emqtt:start_link(#{ + username => <<"api_username">>, clientid => <<"api_clientid">> + }), + {ok, _} = emqtt:connect(Client), + {ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC1), + {ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC2), + Payload = <<"hello">>, + Path = emqx_mgmt_api_test_util:api_path(["publish", "bulk"]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Body = [ + #{topic => ?TOPIC1, payload => Payload}, + #{topic => ?TOPIC2, payload => Payload}, + #{topic => <<"no/subscrbier/topic">>, payload => Payload} + ], + {ok, Response} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body), + ResponseList = decode_json(Response), + ?assertMatch( + [ + #{<<"id">> := _}, + #{<<"id">> := _}, + #{<<"message">> := <<"no_matching_subscribers">>} + ], + ResponseList + ), + ?assertEqual(ok, receive_assert(?TOPIC1, 0, Payload)), + ?assertEqual(ok, receive_assert(?TOPIC2, 0, Payload)), + emqtt:stop(Client). + +t_publish_bulk_dispatch_one_message_invalid_topic({init, Config}) -> + Config; +t_publish_bulk_dispatch_one_message_invalid_topic({'end', _Config}) -> + ok; +t_publish_bulk_dispatch_one_message_invalid_topic(Config) when is_list(Config) -> + Payload = <<"hello">>, + Path = emqx_mgmt_api_test_util:api_path(["publish", "bulk"]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Body = [ + #{topic => ?TOPIC1, payload => Payload}, + #{topic => ?TOPIC2, payload => Payload}, + #{topic => <<"bad/#/topic">>, payload => Payload} + ], + {error, {Summary, _Headers, ResponseBody}} = + emqx_mgmt_api_test_util:request_api( + post, + Path, + "", + Auth, + Body, + #{return_body => true} + ), + ?assertMatch({_, 400, _}, Summary), + ?assertMatch( + #{<<"reason_code">> := ?RC_TOPIC_NAME_INVALID}, + decode_json(ResponseBody) + ). + +t_publish_bulk_dispatch_failure({init, Config}) -> + meck:new(emqx, [no_link, passthrough, no_history]), + meck:expect(emqx, is_running, fun() -> false end), + Config; +t_publish_bulk_dispatch_failure({'end', _Config}) -> + meck:unload(emqx), + ok; +t_publish_bulk_dispatch_failure(Config) when is_list(Config) -> + {ok, Client} = emqtt:start_link(#{ + username => <<"api_username">>, clientid => <<"api_clientid">> + }), + {ok, _} = emqtt:connect(Client), + {ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC1), + {ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC2), + Payload = <<"hello">>, + Path = emqx_mgmt_api_test_util:api_path(["publish", "bulk"]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Body = [ + #{topic => ?TOPIC1, payload => Payload}, + #{topic => ?TOPIC2, payload => Payload}, + #{topic => <<"no/subscrbier/topic">>, payload => Payload} + ], + {error, {Summary, _Headers, ResponseBody}} = + emqx_mgmt_api_test_util:request_api( + post, + Path, + "", + Auth, + Body, + #{return_body => true} + ), + ?assertMatch({_, 503, _}, Summary), + ?assertMatch( + [ + #{<<"reason_code">> := ?RC_IMPLEMENTATION_SPECIFIC_ERROR}, + #{<<"reason_code">> := ?RC_IMPLEMENTATION_SPECIFIC_ERROR}, + #{<<"reason_code">> := ?RC_NO_MATCHING_SUBSCRIBERS} + ], + decode_json(ResponseBody) + ), + emqtt:stop(Client). receive_assert(Topic, Qos, Payload) -> receive @@ -90,3 +319,6 @@ receive_assert(Topic, Qos, Payload) -> after 5000 -> timeout end. + +decode_json(In) -> + emqx_json:decode(In, [return_maps]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index a8b04dc80..1bc29dfee 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -65,8 +65,11 @@ request_api(Method, Url, QueryParams, AuthOrHeaders, []) when "" -> Url; _ -> Url ++ "?" ++ QueryParams end, - do_request_api(Method, {NewUrl, build_http_header(AuthOrHeaders)}); -request_api(Method, Url, QueryParams, AuthOrHeaders, Body) when + do_request_api(Method, {NewUrl, build_http_header(AuthOrHeaders)}, #{}); +request_api(Method, Url, QueryParams, AuthOrHeaders, Body) -> + request_api(Method, Url, QueryParams, AuthOrHeaders, Body, #{}). + +request_api(Method, Url, QueryParams, AuthOrHeaders, Body, Opts) when (Method =:= post) orelse (Method =:= patch) orelse (Method =:= put) orelse @@ -79,10 +82,12 @@ request_api(Method, Url, QueryParams, AuthOrHeaders, Body) when end, do_request_api( Method, - {NewUrl, build_http_header(AuthOrHeaders), "application/json", emqx_json:encode(Body)} + {NewUrl, build_http_header(AuthOrHeaders), "application/json", emqx_json:encode(Body)}, + Opts ). -do_request_api(Method, Request) -> +do_request_api(Method, Request, Opts) -> + ReturnBody = maps:get(return_body, Opts, false), ct:pal("Method: ~p, Request: ~p", [Method, Request]), case httpc:request(Method, Request, [], []) of {error, socket_closed_remotely} -> @@ -91,8 +96,9 @@ do_request_api(Method, Request) -> Code >= 200 andalso Code =< 299 -> {ok, Return}; - {ok, {Reason, _, _} = Error} -> - ct:pal("error: ~p~n", [Error]), + {ok, {Reason, Headers, Body}} when ReturnBody -> + {error, {Reason, Headers, Body}}; + {ok, {Reason, _Headers, _Body}} -> {error, Reason} end. From 72a228c52136f5a3ce1ec3d6e27ed91a7306513c Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 21 Oct 2022 09:53:31 +0800 Subject: [PATCH 067/165] chore: update apps/emqx/i18n/emqx_schema_i18n.conf Co-authored-by: Zaiming (Stone) Shi --- apps/emqx/i18n/emqx_schema_i18n.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index ae9da4579..81e9c96dc 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -869,7 +869,7 @@ When set to true, invalid utf8 strings in for example client ID, topic name, etc mqtt_max_awaiting_rel { desc { - en: """EMQX creates a wait queue in the client's session to accept QoS 2 messages from the client until this QoS 2 message transmission is complete or is discarded due to timeout.
+ en: """For each publisher session, the maximum number of outstanding QoS 2 messages pending on the client to send PUBREL. After reaching this limit, new QoS 2 PUBLISH requests will be rejected with reason code `147(0x93)` until either PUBREL is received or timed out.
After this queue is full, the processing of QoS 2 messages will be stopped.""" zh: """EMQX 会在客户端的会话中创建一个等待队列来接受客户端的 QoS 2 消息,直到这条 QoS 2 消息传输完成或由于超时被丢弃。
该队列满后会停止处理 QoS 2 消息。""" From 6d232fda5c1a2a99f5fc606013e34ddcf97f0933 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 21 Oct 2022 10:27:20 +0800 Subject: [PATCH 068/165] chore(i18n): improve the max_awaiting_rel desc --- apps/emqx/i18n/emqx_schema_i18n.conf | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 81e9c96dc..5e4823992 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -869,10 +869,8 @@ When set to true, invalid utf8 strings in for example client ID, topic name, etc mqtt_max_awaiting_rel { desc { - en: """For each publisher session, the maximum number of outstanding QoS 2 messages pending on the client to send PUBREL. After reaching this limit, new QoS 2 PUBLISH requests will be rejected with reason code `147(0x93)` until either PUBREL is received or timed out.
-After this queue is full, the processing of QoS 2 messages will be stopped.""" - zh: """EMQX 会在客户端的会话中创建一个等待队列来接受客户端的 QoS 2 消息,直到这条 QoS 2 消息传输完成或由于超时被丢弃。
-该队列满后会停止处理 QoS 2 消息。""" + en: """For each publisher session, the maximum number of outstanding QoS 2 messages pending on the client to send PUBREL. After reaching this limit, new QoS 2 PUBLISH requests will be rejected with reason code `147(0x93)` until either PUBREL is received or timed out.""" + zh: """每个发布者的会话中,都存在一个队列来处理客户端发送的 QoS 2 消息。该队列会存储 QoS 2 消息的报文 ID 直到收到客户端的 PUBREL 或超时,达到队列长度的限制后,新的 QoS 2 消息发布会被拒绝,并返回 `147(0x93)` 错误。""" } label: { en: """Max Awaiting PUBREL""" From 1d564ab8d520fe542be5ab29d8447b32c9f7a424 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 21 Oct 2022 14:37:46 +0800 Subject: [PATCH 069/165] test(shared_sub): remove redundant test case --- apps/emqx/test/emqx_shared_sub_SUITE.erl | 72 ------------------------ 1 file changed, 72 deletions(-) diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 45d9b4fad..f2b71af32 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -818,78 +818,6 @@ t_dispatch_qos0(Config) when is_list(Config) -> emqtt:stop(ConnPid2), ok. -t_redispatch_when_kicked({init, Config}) when is_list(Config) -> - Config; -t_redispatch_when_kicked({'end', Config}) when is_list(Config) -> - snabbkaffe:stop(), - ok; -t_redispatch_when_kicked(_) -> - ok = ensure_config(sticky, true), - - Group = <<"group1">>, - Topic = <<"foo/bar">>, - ClientId1 = <<"ClientId1">>, - ClientId2 = <<"ClientId2">>, - Parent = self(), - - %% emqx_cm:kick_session will cause the emqtt to exit abnormally - %% so here need to isolate emqtt with a separate process - MkSub = fun(ClientId) -> - {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}, {auto_ack, false}]), - {ok, _} = emqtt:connect(ConnPid), - {ok, _Props, _ReasonCodes} = emqtt:subscribe(ConnPid, { - <<"$share/", Group/binary, "/foo/bar">>, 1 - }), - - Loop = fun(Self) -> - receive - wait -> - case last_message(<<"hello1">>, [ConnPid], 6000) of - {true, _} -> - Parent ! {got, ClientId}; - _ -> - Parent ! nothing - end, - Self(Self); - stop -> - stop - end - end, - - Loop(Loop) - end, - - Receive = fun() -> - receive - {got, _} = Got -> Got; - nothing -> nothing - after 6000 -> - nothing - end - end, - - Subs = [erlang:spawn(fun() -> MkSub(ClientId) end) || ClientId <- [ClientId1, ClientId2]], - timer:sleep(500), - - ok = snabbkaffe:start_trace(), - - Message = emqx_message:make(ClientId1, 1, Topic, <<"hello1">>), - emqx:publish(Message), - - [Sub ! wait || Sub <- Subs], - Res = Receive(), - ?assertMatch({got, _}, Res), - - {got, ClientId} = Res, - emqx_cm:kick_session(ClientId), - - Trace = snabbkaffe:collect_trace(500), - ?assertMatch([#{reason := kicked}], ?of_kind(ignore_redispatch_shared_messages, Trace)), - - [Sub ! stop || Sub <- Subs], - snabbkaffe:stop(), - ok. - %%-------------------------------------------------------------------- %% help functions %%-------------------------------------------------------------------- From 49dd25cb1438e24603b852e2fab6c711d7aeb802 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 21 Oct 2022 14:38:46 +0800 Subject: [PATCH 070/165] chore: make spellcheck happy --- apps/emqx/i18n/emqx_schema_i18n.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 5e4823992..e0a8e2995 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -869,7 +869,7 @@ When set to true, invalid utf8 strings in for example client ID, topic name, etc mqtt_max_awaiting_rel { desc { - en: """For each publisher session, the maximum number of outstanding QoS 2 messages pending on the client to send PUBREL. After reaching this limit, new QoS 2 PUBLISH requests will be rejected with reason code `147(0x93)` until either PUBREL is received or timed out.""" + en: """For each publisher session, the maximum number of outstanding QoS 2 messages pending on the client to send PUBREL. After reaching this limit, new QoS 2 PUBLISH requests will be rejected with `147(0x93)` until either PUBREL is received or timed out.""" zh: """每个发布者的会话中,都存在一个队列来处理客户端发送的 QoS 2 消息。该队列会存储 QoS 2 消息的报文 ID 直到收到客户端的 PUBREL 或超时,达到队列长度的限制后,新的 QoS 2 消息发布会被拒绝,并返回 `147(0x93)` 错误。""" } label: { From 0f03449b43ae30a9d714ec4c8a194f379c7fabbb Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 21 Oct 2022 16:08:09 +0800 Subject: [PATCH 071/165] chore: improve max_awating_rel_timeout conf docs Co-authored-by: Zaiming (Stone) Shi --- apps/emqx/i18n/emqx_schema_i18n.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index e0a8e2995..d4dae31c7 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -880,8 +880,8 @@ When set to true, invalid utf8 strings in for example client ID, topic name, etc mqtt_await_rel_timeout { desc { - en: """The QoS 2 messages (Client -> Broker) handle flow will be ignored if awaiting PUBREL timeout. This QoS 2 message can be published successfully.""" - zh: """如果等待 PUBREL 超时,QoS 2消息(Client -> Broker)处理流程将被忽略。这条 QoS 2 仍然会成功发布。""" + en: """For client to broker QoS 2 message, the time limit for the broker to wait before the `PUBREL` message is received. The wait is aborted after timed out, and receiving a stale `PUBREL` causes a warning level log. Note, the message is delivered to subscribers before entering the wait for PUBREL.""" + zh: """客户端发布 QoS 2 消息时,服务器等待 `PUBREL` 的最长时延。超过该时长后服务器会放弃等待,如果在这之后收到 PUBREL,服务器将会产生一条告警日志。注意,向订阅客户端转发消息的动作发生在进入等待之前。""" } label: { en: """Max Awaiting PUBREL TIMEOUT""" From 9d650a69b8a5c2a42add026f3d5f367c8b4caee3 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Fri, 21 Oct 2022 10:25:22 +0200 Subject: [PATCH 072/165] fix(gen_rpc): Fix Chinese translation --- apps/emqx_conf/i18n/emqx_conf_schema.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_conf/i18n/emqx_conf_schema.conf b/apps/emqx_conf/i18n/emqx_conf_schema.conf index 6ba57f9ee..101823d5f 100644 --- a/apps/emqx_conf/i18n/emqx_conf_schema.conf +++ b/apps/emqx_conf/i18n/emqx_conf_schema.conf @@ -976,11 +976,11 @@ until the RPC connection is considered lost.""" rpc_insecure_fallback { desc { en: """Enable compatibility with old RPC authentication.""" - zh: """启用与旧RPC认证的兼容性。""" + zh: """兼容旧的无鉴权模式""" } label { en: "RPC insecure fallback" - zh: "RPC不安全的后退" + zh: "向后兼容旧的无鉴权模式" } } From ca052bd0f4559df7b7b647ce1a0257e79eaf8c61 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 21 Oct 2022 17:28:49 +0200 Subject: [PATCH 073/165] docs: Update apps/emqx/i18n/emqx_schema_i18n.conf --- apps/emqx/i18n/emqx_schema_i18n.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index d4dae31c7..94aae19cf 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -880,8 +880,8 @@ When set to true, invalid utf8 strings in for example client ID, topic name, etc mqtt_await_rel_timeout { desc { - en: """For client to broker QoS 2 message, the time limit for the broker to wait before the `PUBREL` message is received. The wait is aborted after timed out, and receiving a stale `PUBREL` causes a warning level log. Note, the message is delivered to subscribers before entering the wait for PUBREL.""" - zh: """客户端发布 QoS 2 消息时,服务器等待 `PUBREL` 的最长时延。超过该时长后服务器会放弃等待,如果在这之后收到 PUBREL,服务器将会产生一条告警日志。注意,向订阅客户端转发消息的动作发生在进入等待之前。""" + en: """For client to broker QoS 2 message, the time limit for the broker to wait before the `PUBREL` message is received. The wait is aborted after timed out, meaning the packet ID is freed for new `PUBLISH` requests. Receiving a stale `PUBREL` causes a warning level log. Note, the message is delivered to subscribers before entering the wait for PUBREL.""" + zh: """客户端发布 QoS 2 消息时,服务器等待 `PUBREL` 的最长时延。超过该时长后服务器会放弃等待,该PACKET ID 会被释放,从而允许后续新的 PUBLISH 消息使用。如果超时后收到 PUBREL,服务器将会产生一条告警日志。注意,向订阅客户端转发消息的动作发生在进入等待之前。""" } label: { en: """Max Awaiting PUBREL TIMEOUT""" From 31c0d73cd5239827a9dc35bc82cdf78aa07a39a3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 20 Oct 2022 18:45:17 +0200 Subject: [PATCH 074/165] chore: split change logs --- CHANGES-5.0.md | 205 ------------------------------------------- changes/v5.0.1-en.md | 53 +++++++++++ changes/v5.0.2-en.md | 18 ++++ changes/v5.0.3-en.md | 12 +++ changes/v5.0.4-en.md | 37 ++++++++ changes/v5.0.5-en.md | 17 ++++ changes/v5.0.6-en.md | 5 ++ changes/v5.0.7-en.md | 12 +++ changes/v5.0.8-en.md | 28 ++++++ changes/v5.0.9-en.md | 34 +++++++ changes/v5.0.9-zh.md | 28 ++++++ 11 files changed, 244 insertions(+), 205 deletions(-) delete mode 100644 CHANGES-5.0.md create mode 100644 changes/v5.0.1-en.md create mode 100644 changes/v5.0.2-en.md create mode 100644 changes/v5.0.3-en.md create mode 100644 changes/v5.0.4-en.md create mode 100644 changes/v5.0.5-en.md create mode 100644 changes/v5.0.6-en.md create mode 100644 changes/v5.0.7-en.md create mode 100644 changes/v5.0.8-en.md create mode 100644 changes/v5.0.9-en.md create mode 100644 changes/v5.0.9-zh.md diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md deleted file mode 100644 index d75766f0b..000000000 --- a/CHANGES-5.0.md +++ /dev/null @@ -1,205 +0,0 @@ -# 5.0.9 - -## Enhancements - -* Add `cert_common_name` and `cert_subject` placeholder support for authz_http and authz_mongo.[#8973](https://github.com/emqx/emqx/pull/8973) -* Use milliseconds internally in emqx_delayed to store the publish time, improving precision.[#9060](https://github.com/emqx/emqx/pull/9060) -* More rigorous checking of flapping to improve stability of the system. [#9136](https://github.com/emqx/emqx/pull/9136) -* No message(s) echo for the message publish APIs [#9155](https://github.com/emqx/emqx/pull/9155) - Prior to this fix, the message publish APIs (`api/v5/publish` and `api/v5/publish/bulk`) echos the message back to the client in HTTP body. - This change fixed it to only send back the message ID. - -## Bug fixes - -* Check ACLs for last will testament topic before publishing the message. [#8930](https://github.com/emqx/emqx/pull/8930) -* Fix GET /listeners API crash When some nodes still in initial configuration. [#9002](https://github.com/emqx/emqx/pull/9002) -* Fix empty variable interpolation in authentication and authorization. Placeholders for undefined variables are rendered now as empty strings and do not cause errors anymore. [#8963](https://github.com/emqx/emqx/pull/8963) -* Fix the latency statistics error of the slow subscription module when `stats_type` is `internal` or `response`. [#8986](https://github.com/emqx/emqx/pull/8986) -* Redispatch shared subscription messages. [#9104](https://github.com/emqx/emqx/pull/9104) - -# 5.0.8 - -## Bug fixes - -* Fix exhook `client.authorize` never being execauted. [#8780](https://github.com/emqx/emqx/pull/8780) -* Fix JWT plugin don't support non-integer timestamp claims. [#8867](https://github.com/emqx/emqx/pull/8867) -* Avoid publishing will message when client fails to auhtenticate. [#8887](https://github.com/emqx/emqx/pull/8887) -* Speed up dispatching of shared subscription messages in a cluster [#8893](https://github.com/emqx/emqx/pull/8893) -* Fix the extra / prefix when CoAP gateway parsing client topics. [#8658](https://github.com/emqx/emqx/pull/8658) -* Speed up updating the configuration, When some nodes in the cluster are down. [#8857](https://github.com/emqx/emqx/pull/8857) -* Fix delayed publish inaccurate caused by os time change. [#8926](https://github.com/emqx/emqx/pull/8926) -* Fix that EMQX can't start when the retainer is disabled [#8911](https://github.com/emqx/emqx/pull/8911) -* Fix that redis authn will deny the unknown users [#8934](https://github.com/emqx/emqx/pull/8934) -* Fix ExProto UDP client keepalive checking error. - This causes the clients to not expire as long as a new UDP packet arrives [#8866](https://github.com/emqx/emqx/pull/8866) -* Fix that MQTT Bridge message payload could be empty string. [#8949](https://github.com/emqx/emqx/pull/8949) - -## Enhancements - -* Print a warning message when boot with the default (insecure) Erlang cookie. [#8905](https://github.com/emqx/emqx/pull/8905) -* Change the `/gateway` API path to plural form. [#8823](https://github.com/emqx/emqx/pull/8823) -* Don't allow updating config items when they already exist in `local-override.conf`. [#8851](https://github.com/emqx/emqx/pull/8851) -* Remove `node.etc_dir` from emqx.conf, because it is never used. - Also allow user to customize the logging directory [#8892](https://github.com/emqx/emqx/pull/8892) -* Added a new API `POST /listeners` for creating listener. [#8876](https://github.com/emqx/emqx/pull/8876) -* Close ExProto client process immediately if it's keepalive timeouted. [#8866](https://github.com/emqx/emqx/pull/8866) -* Upgrade grpc-erl driver to 0.6.7 to support batch operation in sending stream. [#8866](https://github.com/emqx/emqx/pull/8866) - -# 5.0.7 - -## Bug fixes - -* Remove `will_msg` (not used) field from the client API. [#8721](https://github.com/emqx/emqx/pull/8721) -* Fix `$queue` topic name error in management API return. [#8728](https://github.com/emqx/emqx/pull/8728) -* Fix race condition which may cause `client.connected` and `client.disconnected` out of order. [#8625](https://github.com/emqx/emqx/pull/8625) -* Fix quic listener default idle timeout's type. [#8826](https://github.com/emqx/emqx/pull/8826) - -## Enhancements - -* Do not auto-populate default SSL cipher suites, so that the configs are less bloated. [#8769](https://github.com/emqx/emqx/pull/8769) - -# 5.0.6 - -## Bug fixes - -* Upgrade Dashboard version to fix an issue where the node status was not displayed correctly. [#8771](https://github.com/emqx/emqx/pull/8771) - -# 5.0.5 - -## Bug fixes - -* Allow changing the license type from key to file (and vice-versa). [#8598](https://github.com/emqx/emqx/pull/8598) -* Add back http connector config keys `max_retries` `retry_interval` as deprecated fields [#8672](https://github.com/emqx/emqx/issues/8672) - This caused upgrade failure in 5.0.4, because it would fail to boot on configs created from older version. - -## Enhancements - -* Add `bootstrap_users_file` configuration to add default Dashboard username list, which is only added when EMQX is first started. -* The license is now copied to all nodes in the cluster when it's reloaded. [#8598](https://github.com/emqx/emqx/pull/8598) -* Added a HTTP API to manage licenses. [#8610](https://github.com/emqx/emqx/pull/8610) -* Updated `/nodes` API node_status from `Running/Stopped` to `running/stopped`. [#8642](https://github.com/emqx/emqx/pull/8642) -* Improve handling of placeholder interpolation errors [#8635](https://github.com/emqx/emqx/pull/8635) -* Better logging on unknown object IDs. [#8670](https://github.com/emqx/emqx/pull/8670) -* The bind option support `:1883` style. [#8758](https://github.com/emqx/emqx/pull/8758) - -# 5.0.4 - -## Bug fixes - -* The `data/configs/cluster-override.conf` is cleared to 0KB if `hocon_pp:do/2` failed [commits/71f64251](https://github.com/emqx/emqx/pull/8443/commits/71f642518a683cc91a32fd542aafaac6ef915720) -* Improve the health_check for webhooks. - Prior to this change, the webhook only checks the connectivity of the TCP port using `gen_tcp:connect/2`, so - if it's a HTTPs server, we didn't check if TLS handshake was successful. - [commits/6b45d2ea](https://github.com/emqx/emqx/commit/6b45d2ea9fde6d3b4a5b007f7a8c5a1c573d141e) -* The `created_at` field of rules is missing after emqx restarts. [commits/5fc09e6b](https://github.com/emqx/emqx/commit/5fc09e6b950c340243d7be627a0ce1700691221c) -* The rule engine's jq function now works even when the path to the EMQX install dir contains spaces [jq#35](https://github.com/emqx/jq/pull/35) [#8455](https://github.com/emqx/emqx/pull/8455) -* Avoid applying any ACL checks on superusers [#8452](https://github.com/emqx/emqx/pull/8452) -* Fix statistics related system topic name error -* Fix AuthN JWKS SSL schema. Using schema in `emqx_schema`. [#8458](https://github.com/emqx/emqx/pull/8458) -* `sentinel` field should be required when AuthN/AuthZ Redis using sentinel mode. [#8458](https://github.com/emqx/emqx/pull/8458) -* Fix bad swagger format. [#8517](https://github.com/emqx/emqx/pull/8517) -* Fix `chars_limit` is not working when `formatter` is `json`. [#8518](http://github.com/emqx/emqx/pull/8518) -* Ensuring that exhook dispatches the client events are sequential. [#8530](https://github.com/emqx/emqx/pull/8530) -* Avoid using RocksDB backend for persistent sessions when such backend is unavailable. [#8528](https://github.com/emqx/emqx/pull/8528) -* Fix AuthN `cert_subject` and `cert_common_name` placeholder rendering failure. [#8531](https://github.com/emqx/emqx/pull/8531) -* Support listen on an IPv6 address, e.g: [::1]:1883 or ::1:1883. [#8547](https://github.com/emqx/emqx/pull/8547) -* GET '/rules' support for pagination and fuzzy search. [#8472](https://github.com/emqx/emqx/pull/8472) - **‼️ Note** : The previous API only returns array: `[RuleObj1,RuleObj2]`, after updating, it will become - `{"data": [RuleObj1,RuleObj2], "meta":{"count":2, "limit":100, "page":1}`, - which will carry the paging meta information. -* Fix the issue that webhook leaks TCP connections. [ehttpc#34](https://github.com/emqx/ehttpc/pull/34), [#8580](https://github.com/emqx/emqx/pull/8580) - -## Enhancements - -* Improve the dashboard listener startup log, the listener name is no longer spliced with port information, - and the colon(:) is no longer displayed when IP is not specified. [#8480](https://github.com/emqx/emqx/pull/8480) -* Remove `/configs/listeners` API, use `/listeners/` instead. [#8485](https://github.com/emqx/emqx/pull/8485) -* Optimize performance of builtin database operations in processes with long message queue [#8439](https://github.com/emqx/emqx/pull/8439) -* Improve authentication tracing. [#8554](https://github.com/emqx/emqx/pull/8554) -* Standardize the '/listeners' and `/gateway//listeners` API fields. - It will introduce some incompatible updates, see [#8571](https://github.com/emqx/emqx/pull/8571) -* Add option to perform GC on connection process after TLS/SSL handshake is performed. [#8637](https://github.com/emqx/emqx/pull/8637) - -# 5.0.3 - -## Bug fixes - -* Websocket listener failed to read headers `X-Forwarded-For` and `X-Forwarded-Port` [#8415](https://github.com/emqx/emqx/pull/8415) -* Deleted `cluster_singleton` from MQTT bridge config document. This config is no longer applicable in 5.0 [#8407](https://github.com/emqx/emqx/pull/8407) -* Fix `emqx/emqx:latest` docker image publish to use the Erlang flavor, but not Elixir flavor [#8414](https://github.com/emqx/emqx/pull/8414) -* Changed the `exp` field in JWT auth to be optional rather than required to fix backwards compatability with 4.X releases. [#8425](https://github.com/emqx/emqx/pull/8425) - -## Enhancements - -* Improve the speed of dashboard's HTTP API routing rule generation, which sometimes causes timeout [#8438](https://github.com/emqx/emqx/pull/8438) - -# 5.0.2 - -Announcement: EMQX team has decided to stop supporting relup for opensource edition. -Going forward, it will be an enterprise-only feature. - -Main reason: relup requires carefully crafted upgrade instructions from ALL previous versions. - -For example, 4.3 is now at 4.3.16, we have `4.3.0->4.3.16`, `4.3.1->4.3.16`, ... 16 such upgrade paths in total to maintain. -This had been the biggest obstacle for EMQX team to act agile enough in delivering enhancements and fixes. - -## Enhancements - -## Bug fixes - -* Fixed a typo in `bin/emqx` which affects MacOs release when trying to enable Erlang distribution over TLS [#8398](https://github.com/emqx/emqx/pull/8398) -* Restricted shell was accidentally disabled in 5.0.1, it has been added back. [#8396](https://github.com/emqx/emqx/pull/8396) - -# 5.0.1 - -5.0.1 is built on [Erlang/OTP 24.2.1-1](https://github.com/emqx/otp/tree/OTP-24.2.1-1). Same as 5.0.0. - -5.0.0 (like 4.4.x) had Erlang/OTP version number in the package name. -This is because we wanted to release different flavor packages (on different Elixir/Erlang/OTP platforms). - -However the long package names also causes confusion, as users may not know which to choose if there were more than -one presented at the same time. - -Going forward, (starting from 5.0.1), packages will be released in both default (short) and flavored (long) package names. - -For example: `emqx-5.0.1-otp24.2.1-1-ubuntu20.04-amd64.tar.gz`, -but only the default one is presented to the users: `emqx-5.0.1-ubuntu20.04-amd64.tar.gz`. - -In case anyone wants to try a different flavor package, it can be downlowded from the public s3 bucket, -for example: -https://s3.us-west-2.amazonaws.com/packages.emqx/emqx-ce/v5.0.1/emqx-5.0.1-otp24.2.1-1-ubuntu20.04-arm64.tar.gz - -Exceptions: - -* Windows package is always presented with short name (currently on Erlang/OTP 24.2.1). -* Elixir package name is flavored with both Elixir and Erlang/OTP version numbers, - for example: `emqx-5.0.1-elixir1.13.4-otp24.2.1-1-ubuntu20.04-amd64.tar.gz` - -## Enhancements - -* Removed management API auth for prometheus scraping endpoint /api/v5/prometheus/stats [#8299](https://github.com/emqx/emqx/pull/8299) -* Added more TCP options for exhook (gRPC) connections. [#8317](https://github.com/emqx/emqx/pull/8317) -* HTTP Servers used for authentication and authorization will now indicate the result via the response body. [#8374](https://github.com/emqx/emqx/pull/8374) [#8377](https://github.com/emqx/emqx/pull/8377) -* Bulk subscribe/unsubscribe APIs [#8356](https://github.com/emqx/emqx/pull/8356) -* Added exclusive subscription [#8315](https://github.com/emqx/emqx/pull/8315) -* Provide authentication counter metrics [#8352](https://github.com/emqx/emqx/pull/8352) [#8375](https://github.com/emqx/emqx/pull/8375) -* Do not allow admin user self-deletion [#8286](https://github.com/emqx/emqx/pull/8286) -* After restart, ensure to copy `cluster-override.conf` from the clustered node which has the greatest `tnxid`. [#8333](https://github.com/emqx/emqx/pull/8333) - -## Bug fixes - -* A bug fix ported from 4.x: allow deleting subscriptions from `client.subscribe` hookpoint callback result. [#8304](https://github.com/emqx/emqx/pull/8304) [#8347](https://github.com/emqx/emqx/pull/8377) -* Fixed Erlang distribution over TLS [#8309](https://github.com/emqx/emqx/pull/8309) -* Made possible to override authentication configs from environment variables [#8323](https://github.com/emqx/emqx/pull/8309) -* Made authentication passwords in Mnesia database backward compatible to 4.x, so we can support data migration better. [#8351](https://github.com/emqx/emqx/pull/8351) -* Fix plugins upload for rpm/deb installations [#8379](https://github.com/emqx/emqx/pull/8379) -* Sync data/authz/acl.conf and data/certs from clustered nodes after a new node joins the cluster [#8369](https://github.com/emqx/emqx/pull/8369) -* Ensure auto-retry of failed resources [#8371](https://github.com/emqx/emqx/pull/8371) -* Fix the issue that the count of `packets.connack.auth_error` is inaccurate when the client uses a protocol version below MQTT v5.0 to access [#8178](https://github.com/emqx/emqx/pull/8178) - -## Others - -* Rate limiter interface is hidden so far, it's subject to a UX redesign. -* QUIC library upgraded to 0.0.14. -* Now the default packages will be released withot otp version number in the package name. -* Renamed config exmpale file name in `etc` dir. diff --git a/changes/v5.0.1-en.md b/changes/v5.0.1-en.md new file mode 100644 index 000000000..f3c301637 --- /dev/null +++ b/changes/v5.0.1-en.md @@ -0,0 +1,53 @@ +# v5.0.1 + +5.0.1 is built on [Erlang/OTP 24.2.1-1](https://github.com/emqx/otp/tree/OTP-24.2.1-1). Same as 5.0.0. + +5.0.0 (like 4.4.x) had Erlang/OTP version number in the package name. +This is because we wanted to release different flavor packages (on different Elixir/Erlang/OTP platforms). + +However the long package names also causes confusion, as users may not know which to choose if there were more than +one presented at the same time. + +Going forward, (starting from 5.0.1), packages will be released in both default (short) and flavored (long) package names. + +For example: `emqx-5.0.1-otp24.2.1-1-ubuntu20.04-amd64.tar.gz`, +but only the default one is presented to the users: `emqx-5.0.1-ubuntu20.04-amd64.tar.gz`. + +In case anyone wants to try a different flavor package, it can be downlowded from the public s3 bucket, +for example: +https://s3.us-west-2.amazonaws.com/packages.emqx/emqx-ce/v5.0.1/emqx-5.0.1-otp24.2.1-1-ubuntu20.04-arm64.tar.gz + +Exceptions: + +* Windows package is always presented with short name (currently on Erlang/OTP 24.2.1). +* Elixir package name is flavored with both Elixir and Erlang/OTP version numbers, + for example: `emqx-5.0.1-elixir1.13.4-otp24.2.1-1-ubuntu20.04-amd64.tar.gz` + +## Enhancements + +* Removed management API auth for prometheus scraping endpoint /api/v5/prometheus/stats [#8299](https://github.com/emqx/emqx/pull/8299) +* Added more TCP options for exhook (gRPC) connections. [#8317](https://github.com/emqx/emqx/pull/8317) +* HTTP Servers used for authentication and authorization will now indicate the result via the response body. [#8374](https://github.com/emqx/emqx/pull/8374) [#8377](https://github.com/emqx/emqx/pull/8377) +* Bulk subscribe/unsubscribe APIs [#8356](https://github.com/emqx/emqx/pull/8356) +* Added exclusive subscription [#8315](https://github.com/emqx/emqx/pull/8315) +* Provide authentication counter metrics [#8352](https://github.com/emqx/emqx/pull/8352) [#8375](https://github.com/emqx/emqx/pull/8375) +* Do not allow admin user self-deletion [#8286](https://github.com/emqx/emqx/pull/8286) +* After restart, ensure to copy `cluster-override.conf` from the clustered node which has the greatest `tnxid`. [#8333](https://github.com/emqx/emqx/pull/8333) + +## Bug fixes + +* A bug fix ported from 4.x: allow deleting subscriptions from `client.subscribe` hookpoint callback result. [#8304](https://github.com/emqx/emqx/pull/8304) [#8347](https://github.com/emqx/emqx/pull/8377) +* Fixed Erlang distribution over TLS [#8309](https://github.com/emqx/emqx/pull/8309) +* Made possible to override authentication configs from environment variables [#8323](https://github.com/emqx/emqx/pull/8309) +* Made authentication passwords in Mnesia database backward compatible to 4.x, so we can support data migration better. [#8351](https://github.com/emqx/emqx/pull/8351) +* Fix plugins upload for rpm/deb installations [#8379](https://github.com/emqx/emqx/pull/8379) +* Sync data/authz/acl.conf and data/certs from clustered nodes after a new node joins the cluster [#8369](https://github.com/emqx/emqx/pull/8369) +* Ensure auto-retry of failed resources [#8371](https://github.com/emqx/emqx/pull/8371) +* Fix the issue that the count of `packets.connack.auth_error` is inaccurate when the client uses a protocol version below MQTT v5.0 to access [#8178](https://github.com/emqx/emqx/pull/8178) + +## Others + +* Rate limiter interface is hidden so far, it's subject to a UX redesign. +* QUIC library upgraded to 0.0.14. +* Now the default packages will be released withot otp version number in the package name. +* Renamed config exmpale file name in `etc` dir. diff --git a/changes/v5.0.2-en.md b/changes/v5.0.2-en.md new file mode 100644 index 000000000..92e15fe4f --- /dev/null +++ b/changes/v5.0.2-en.md @@ -0,0 +1,18 @@ +# v5.0.2 + +Announcement: EMQX team has decided to stop supporting relup for opensource edition. +Going forward, it will be an enterprise-only feature. + +Main reason: relup requires carefully crafted upgrade instructions from ALL previous versions. + +For example, 4.3 is now at 4.3.16, we have `4.3.0->4.3.16`, `4.3.1->4.3.16`, ... 16 such upgrade paths in total to maintain. +This had been the biggest obstacle for EMQX team to act agile enough in delivering enhancements and fixes. + +## Enhancements + +## Bug fixes + +* Fixed a typo in `bin/emqx` which affects MacOs release when trying to enable Erlang distribution over TLS [#8398](https://github.com/emqx/emqx/pull/8398) +* Restricted shell was accidentally disabled in 5.0.1, it has been added back. [#8396](https://github.com/emqx/emqx/pull/8396) + + diff --git a/changes/v5.0.3-en.md b/changes/v5.0.3-en.md new file mode 100644 index 000000000..df09e1db5 --- /dev/null +++ b/changes/v5.0.3-en.md @@ -0,0 +1,12 @@ +# v5.0.3 + +## Bug fixes + +* Websocket listener failed to read headers `X-Forwarded-For` and `X-Forwarded-Port` [#8415](https://github.com/emqx/emqx/pull/8415) +* Deleted `cluster_singleton` from MQTT bridge config document. This config is no longer applicable in 5.0 [#8407](https://github.com/emqx/emqx/pull/8407) +* Fix `emqx/emqx:latest` docker image publish to use the Erlang flavor, but not Elixir flavor [#8414](https://github.com/emqx/emqx/pull/8414) +* Changed the `exp` field in JWT auth to be optional rather than required to fix backwards compatability with 4.X releases. [#8425](https://github.com/emqx/emqx/pull/8425) + +## Enhancements + +* Improve the speed of dashboard's HTTP API routing rule generation, which sometimes causes timeout [#8438](https://github.com/emqx/emqx/pull/8438) diff --git a/changes/v5.0.4-en.md b/changes/v5.0.4-en.md new file mode 100644 index 000000000..a2531acd1 --- /dev/null +++ b/changes/v5.0.4-en.md @@ -0,0 +1,37 @@ +# v5.0.4 + +## Enhancements + +* Improve the dashboard listener startup log, the listener name is no longer spliced with port information, + and the colon(:) is no longer displayed when IP is not specified. [#8480](https://github.com/emqx/emqx/pull/8480) +* Remove `/configs/listeners` API, use `/listeners/` instead. [#8485](https://github.com/emqx/emqx/pull/8485) +* Optimize performance of builtin database operations in processes with long message queue [#8439](https://github.com/emqx/emqx/pull/8439) +* Improve authentication tracing. [#8554](https://github.com/emqx/emqx/pull/8554) +* Standardize the '/listeners' and `/gateway//listeners` API fields. + It will introduce some incompatible updates, see [#8571](https://github.com/emqx/emqx/pull/8571) +* Add option to perform GC on connection process after TLS/SSL handshake is performed. [#8637](https://github.com/emqx/emqx/pull/8637) + +## Bug fixes + +* The `data/configs/cluster-override.conf` is cleared to 0KB if `hocon_pp:do/2` failed [commits/71f64251](https://github.com/emqx/emqx/pull/8443/commits/71f642518a683cc91a32fd542aafaac6ef915720) +* Improve the health_check for webhooks. + Prior to this change, the webhook only checks the connectivity of the TCP port using `gen_tcp:connect/2`, so + if it's a HTTPs server, we didn't check if TLS handshake was successful. + [commits/6b45d2ea](https://github.com/emqx/emqx/commit/6b45d2ea9fde6d3b4a5b007f7a8c5a1c573d141e) +* The `created_at` field of rules is missing after emqx restarts. [commits/5fc09e6b](https://github.com/emqx/emqx/commit/5fc09e6b950c340243d7be627a0ce1700691221c) +* The rule engine's jq function now works even when the path to the EMQX install dir contains spaces [jq#35](https://github.com/emqx/jq/pull/35) [#8455](https://github.com/emqx/emqx/pull/8455) +* Avoid applying any ACL checks on superusers [#8452](https://github.com/emqx/emqx/pull/8452) +* Fix statistics related system topic name error +* Fix AuthN JWKS SSL schema. Using schema in `emqx_schema`. [#8458](https://github.com/emqx/emqx/pull/8458) +* `sentinel` field should be required when AuthN/AuthZ Redis using sentinel mode. [#8458](https://github.com/emqx/emqx/pull/8458) +* Fix bad swagger format. [#8517](https://github.com/emqx/emqx/pull/8517) +* Fix `chars_limit` is not working when `formatter` is `json`. [#8518](http://github.com/emqx/emqx/pull/8518) +* Ensuring that exhook dispatches the client events are sequential. [#8530](https://github.com/emqx/emqx/pull/8530) +* Avoid using RocksDB backend for persistent sessions when such backend is unavailable. [#8528](https://github.com/emqx/emqx/pull/8528) +* Fix AuthN `cert_subject` and `cert_common_name` placeholder rendering failure. [#8531](https://github.com/emqx/emqx/pull/8531) +* Support listen on an IPv6 address, e.g: [::1]:1883 or ::1:1883. [#8547](https://github.com/emqx/emqx/pull/8547) +* GET '/rules' support for pagination and fuzzy search. [#8472](https://github.com/emqx/emqx/pull/8472) + **‼️ Note** : The previous API only returns array: `[RuleObj1,RuleObj2]`, after updating, it will become + `{"data": [RuleObj1,RuleObj2], "meta":{"count":2, "limit":100, "page":1}`, + which will carry the paging meta information. +* Fix the issue that webhook leaks TCP connections. [ehttpc#34](https://github.com/emqx/ehttpc/pull/34), [#8580](https://github.com/emqx/emqx/pull/8580) diff --git a/changes/v5.0.5-en.md b/changes/v5.0.5-en.md new file mode 100644 index 000000000..9c462545d --- /dev/null +++ b/changes/v5.0.5-en.md @@ -0,0 +1,17 @@ +# v5.0.5 + +## Enhancements + +* Add `bootstrap_users_file` configuration to add default Dashboard username list, which is only added when EMQX is first started. +* The license is now copied to all nodes in the cluster when it's reloaded. [#8598](https://github.com/emqx/emqx/pull/8598) +* Added a HTTP API to manage licenses. [#8610](https://github.com/emqx/emqx/pull/8610) +* Updated `/nodes` API node_status from `Running/Stopped` to `running/stopped`. [#8642](https://github.com/emqx/emqx/pull/8642) +* Improve handling of placeholder interpolation errors [#8635](https://github.com/emqx/emqx/pull/8635) +* Better logging on unknown object IDs. [#8670](https://github.com/emqx/emqx/pull/8670) +* The bind option support `:1883` style. [#8758](https://github.com/emqx/emqx/pull/8758) + +## Bug fixes + +* Allow changing the license type from key to file (and vice-versa). [#8598](https://github.com/emqx/emqx/pull/8598) +* Add back http connector config keys `max_retries` `retry_interval` as deprecated fields [#8672](https://github.com/emqx/emqx/issues/8672) + This caused upgrade failure in 5.0.4, because it would fail to boot on configs created from older version. diff --git a/changes/v5.0.6-en.md b/changes/v5.0.6-en.md new file mode 100644 index 000000000..342cb67e2 --- /dev/null +++ b/changes/v5.0.6-en.md @@ -0,0 +1,5 @@ +# v5.0.6 + +## Bug fixes + +* Upgrade Dashboard version to fix an issue where the node status was not displayed correctly. [#8771](https://github.com/emqx/emqx/pull/8771) diff --git a/changes/v5.0.7-en.md b/changes/v5.0.7-en.md new file mode 100644 index 000000000..cfda78fb1 --- /dev/null +++ b/changes/v5.0.7-en.md @@ -0,0 +1,12 @@ +# v5.0.7 + +## Enhancements + +* Do not auto-populate default SSL cipher suites, so that the configs are less bloated. [#8769](https://github.com/emqx/emqx/pull/8769) + +## Bug fixes + +* Remove `will_msg` (not used) field from the client API. [#8721](https://github.com/emqx/emqx/pull/8721) +* Fix `$queue` topic name error in management API return. [#8728](https://github.com/emqx/emqx/pull/8728) +* Fix race condition which may cause `client.connected` and `client.disconnected` out of order. [#8625](https://github.com/emqx/emqx/pull/8625) +* Fix quic listener default idle timeout's type. [#8826](https://github.com/emqx/emqx/pull/8826) diff --git a/changes/v5.0.8-en.md b/changes/v5.0.8-en.md new file mode 100644 index 000000000..e05c3b89d --- /dev/null +++ b/changes/v5.0.8-en.md @@ -0,0 +1,28 @@ +# v5.0.8 + +## Enhancements + +* Print a warning message when boot with the default (insecure) Erlang cookie. [#8905](https://github.com/emqx/emqx/pull/8905) +* Change the `/gateway` API path to plural form. [#8823](https://github.com/emqx/emqx/pull/8823) +* Don't allow updating config items when they already exist in `local-override.conf`. [#8851](https://github.com/emqx/emqx/pull/8851) +* Remove `node.etc_dir` from emqx.conf, because it is never used. + Also allow user to customize the logging directory [#8892](https://github.com/emqx/emqx/pull/8892) +* Added a new API `POST /listeners` for creating listener. [#8876](https://github.com/emqx/emqx/pull/8876) +* Close ExProto client process immediately if it's keepalive timeouted. [#8866](https://github.com/emqx/emqx/pull/8866) +* Upgrade grpc-erl driver to 0.6.7 to support batch operation in sending stream. [#8866](https://github.com/emqx/emqx/pull/8866) + + +## Bug fixes + +* Fix exhook `client.authorize` never being execauted. [#8780](https://github.com/emqx/emqx/pull/8780) +* Fix JWT plugin don't support non-integer timestamp claims. [#8867](https://github.com/emqx/emqx/pull/8867) +* Avoid publishing will message when client fails to auhtenticate. [#8887](https://github.com/emqx/emqx/pull/8887) +* Speed up dispatching of shared subscription messages in a cluster [#8893](https://github.com/emqx/emqx/pull/8893) +* Fix the extra / prefix when CoAP gateway parsing client topics. [#8658](https://github.com/emqx/emqx/pull/8658) +* Speed up updating the configuration, When some nodes in the cluster are down. [#8857](https://github.com/emqx/emqx/pull/8857) +* Fix delayed publish inaccurate caused by os time change. [#8926](https://github.com/emqx/emqx/pull/8926) +* Fix that EMQX can't start when the retainer is disabled [#8911](https://github.com/emqx/emqx/pull/8911) +* Fix that redis authn will deny the unknown users [#8934](https://github.com/emqx/emqx/pull/8934) +* Fix ExProto UDP client keepalive checking error. + This causes the clients to not expire as long as a new UDP packet arrives [#8866](https://github.com/emqx/emqx/pull/8866) +* Fix that MQTT Bridge message payload could be empty string. [#8949](https://github.com/emqx/emqx/pull/8949) diff --git a/changes/v5.0.9-en.md b/changes/v5.0.9-en.md new file mode 100644 index 000000000..c6d0b5192 --- /dev/null +++ b/changes/v5.0.9-en.md @@ -0,0 +1,34 @@ +# v5.0.9 + +## Enhancements + +- Add `cert_common_name` and `cert_subject` placeholder support for authz_http and authz_mongo [#8973](https://github.com/emqx/emqx/pull/8973). + +- Use milliseconds internally in emqx_delayed to store the publish time, improving precision [#9060](https://github.com/emqx/emqx/pull/9060). + +- More rigorous checking of flapping to improve stability of the system [#9136](https://github.com/emqx/emqx/pull/9136). + +- No message(s) echo for the message publish APIs [#9155](https://github.com/emqx/emqx/pull/9155). + Prior to this fix, the message publish APIs (`api/v5/publish` and `api/v5/publish/bulk`) echos the message back to the client in HTTP body. + This change fixed it to only send back the message ID. + +## Bug fixes + +- Check ACLs for last will testament topic before publishing the message [#8930](https://github.com/emqx/emqx/pull/8930). + +- Fix GET /listeners API crash when some nodes (in a cluster) is still loading the configs [#9002](https://github.com/emqx/emqx/pull/9002). + +- Fix empty variable interpolation in authentication and authorization [#8963](https://github.com/emqx/emqx/pull/8963). + Placeholders for undefined variables are rendered now as empty strings and do not cause errors anymore. + +- Fix the latency statistics error of the slow subscription stats [#8986](https://github.com/emqx/emqx/pull/8986). + Prior to this change when `stats_type` is `internal` or `response`, the begin time stamp was taken at wrong precision. + +- Fix shared subscription message re-dispatches [#9104](https://github.com/emqx/emqx/pull/9104). + - When discarding QoS 2 inflight messages, there were excessive logs + - For wildcard deliveries, the re-dispatch used the wrong topic (the publishing topic, + but not the subscribing topic), caused messages to be lost when dispatching. + +- Upgrade http client `gun` from 1.3.7 to [1.3.9](https://github.com/emqx/gun/tree/1.3.9) + Prior to this fix, long-lived HTTPS connections for HTTP auth or webhook integrations + may stall indefinitely, causing massive timeouts for HTTP requests. diff --git a/changes/v5.0.9-zh.md b/changes/v5.0.9-zh.md new file mode 100644 index 000000000..460d79941 --- /dev/null +++ b/changes/v5.0.9-zh.md @@ -0,0 +1,28 @@ +# v5.0.9 + +## 增强 + +- 为 `authz_http` 和 `authz_mongo` 增加了 `cert_common_name` 和 `cert_subject` 两个占位符 [#8973](https://github.com/emqx/emqx/pull/8973)。 + +- 统一使用 Erlang 虚拟机的时间,而不是系统时间,可以避免系统时间修改后导致的延迟发布不准确问题 [#9060](https://github.com/emqx/emqx/pull/9060)。 + +- 更严格的 flapping 检测,认证失败等也会进行计数 [#9136](https://github.com/emqx/emqx/pull/9136)。 + +## 修复 + +- 遗嘱消息发布前进行 ACL 检查 [#8930](https://github.com/emqx/emqx/pull/8930)。 + +- 在集群环境下,当有节点还没有完全初始化好配置时,`GET /listeners` 可能会返回 HTTP 500 的错误 [#9002](https://github.com/emqx/emqx/pull/9002)。 + +- 认证和鉴权的占位符替换中,如果没有找到匹配的值,使用空字符串代替,而不是抛出一异常 [#8963](https://github.com/emqx/emqx/pull/8963)。 + +- 慢订阅统计中时间单位用错的问题 [#8986](https://github.com/emqx/emqx/pull/8986)。 + 当统计类型(`stats_type`)是 `internal` 或者 `response` 时,起始时间戳的精度使用错误。 + +- 共享订阅消息重新派发 [#9104](https://github.com/emqx/emqx/pull/9104)。 + - 当 QoS 2 的 inflight 消息被丢弃时,产生了大量的 warning 日志,修复后不再打印。 + - 通配符订阅的共享订阅消息在重新派发时,使用了消息发布时的主题,而不是订阅的通配符主题选择 + 订阅组中的其他成员,导致转发失败。 + +- HTTP 客户端 (`gun`) 从 1.3.7 升级到 [1.3.9](https://github.com/emqx/gun/tree/1.3.9)。 + 此次修复前,HTTP 认证和 webhook 等使用 HTTPS 客户端长连接的后端可能会进入一个无限等待状态,导致大量超时发生。 From 11daaa08e28992ae2d981c11d52643d48dffe022 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 24 Oct 2022 09:42:25 +0200 Subject: [PATCH 075/165] chore: bump version numbers to 5.0.9 --- apps/emqx/include/emqx_release.hrl | 2 +- deploy/charts/emqx/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index ad312e05d..caf70bf5a 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.8"). +-define(EMQX_RELEASE_CE, "5.0.9"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.0-alpha.1"). diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index fa265e663..c46e95c9b 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.8 +version: 5.0.9 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.8 +appVersion: 5.0.9 From e3240b52a89ed7300f44198cbeb442f78561cb80 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 24 Oct 2022 11:48:17 +0200 Subject: [PATCH 076/165] chore: bump app versions --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_authn/src/emqx_authn.app.src | 2 +- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_management/src/emqx_management.app.src | 2 +- apps/emqx_modules/src/emqx_modules.app.src | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index cc1e7eaea..5d2d8eb2c 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.9"}, + {vsn, "5.0.10"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index bdb70cd4e..992e1452b 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, ehttpc, epgsql, mysql, jose]}, diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index c0d5e1f44..2b6efe639 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 5f8b30bf4..b91b6a8b1 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.6"}, + {vsn, "5.0.7"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]}, diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index 4a1644152..e2a142a99 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.5"}, + {vsn, "5.0.6"}, {modules, []}, {applications, [kernel, stdlib, emqx]}, {mod, {emqx_modules_app, []}}, From 67da0a1479bdbb5973a2698ceda501f44641d697 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 24 Oct 2022 11:48:49 +0200 Subject: [PATCH 077/165] chore: bump chart version to 5.0.10 --- deploy/charts/emqx/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index c46e95c9b..b42ca84d0 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.9 +version: 5.0.10 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.9 +appVersion: 5.0.10 From 637bc886bb39c6319b0f34892614036cb4d9ee22 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 24 Oct 2022 11:49:20 +0200 Subject: [PATCH 078/165] chore: bump release version to 5.0.10 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index caf70bf5a..1335ed700 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.9"). +-define(EMQX_RELEASE_CE, "5.0.10"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.0-alpha.1"). From 93195f7b5bb841e5bd25f8b9a99643d899b71b11 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 24 Oct 2022 11:53:51 +0200 Subject: [PATCH 079/165] chore: fix introduced_in for new proto APIs --- apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl | 2 +- apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl b/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl index b9f57b45c..8502b81d9 100644 --- a/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl +++ b/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl @@ -30,7 +30,7 @@ -include_lib("emqx/include/bpapi.hrl"). introduced_in() -> - "5.0.9". + "5.0.10". -spec get_trace_size([node()]) -> emqx_rpc:multicall_result(#{{node(), file:name_all()} => non_neg_integer()}). diff --git a/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl b/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl index 866b2cd40..9bbf35720 100644 --- a/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl +++ b/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl @@ -30,7 +30,7 @@ -define(TIMEOUT, 15000). introduced_in() -> - "5.0.9". + "5.0.10". -spec get_delayed_message(node(), binary()) -> emqx_delayed:with_id_return(map()) | emqx_rpc:badrpc(). From 997a262c96d91c538a82786a80ebb3514b4dcac3 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 21 Oct 2022 15:57:30 -0300 Subject: [PATCH 080/165] fix(mgmt_api): return 503 when emqx is not running in `/status` (5.0) --- .../src/emqx_mgmt_api_status.erl | 7 +- .../test/emqx_mgmt_api_status_SUITE.erl | 88 ++++++++++++++++++- changes/v5.0.10-en.md | 2 + changes/v5.0.10-zh.md | 2 + 4 files changed, 94 insertions(+), 5 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_status.erl b/apps/emqx_management/src/emqx_mgmt_api_status.erl index 68bad03e4..37a8274f8 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_status.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_status.erl @@ -45,7 +45,12 @@ running_status() -> BrokerStatus = broker_status(), AppStatus = application_status(), Body = io_lib:format("Node ~ts is ~ts~nemqx is ~ts", [node(), BrokerStatus, AppStatus]), - {200, #{<<"content-type">> => <<"text/plain">>}, list_to_binary(Body)}; + StatusCode = + case AppStatus of + running -> 200; + not_running -> 503 + end, + {StatusCode, #{<<"content-type">> => <<"text/plain">>}, list_to_binary(Body)}; false -> {503, #{<<"retry-after">> => <<"15">>}, <<>>} end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl index b725e37b2..4051bd62c 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl @@ -20,6 +20,12 @@ -include_lib("eunit/include/eunit.hrl"). +-define(HOST, "http://127.0.0.1:18083/"). + +%%--------------------------------------------------------------------------------------- +%% CT boilerplate +%%--------------------------------------------------------------------------------------- + all() -> emqx_common_test_helpers:all(?MODULE). @@ -30,8 +36,82 @@ init_per_suite(Config) -> end_per_suite(_) -> emqx_mgmt_api_test_util:end_suite(). -t_status(_Config) -> - Path = emqx_mgmt_api_test_util:api_path_without_base_path(["/status"]), - Status = io_lib:format("Node ~ts is ~ts~nemqx is ~ts", [node(), started, running]), - {ok, Status} = emqx_mgmt_api_test_util:request_api(get, Path), +init_per_testcase(t_status_not_ok, Config) -> + ok = application:stop(emqx), + Config; +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(t_status_not_ok, _Config) -> + {ok, _} = application:ensure_all_started(emqx), + ok; +end_per_testcase(_TestCase, _Config) -> + ok. + +%%--------------------------------------------------------------------------------------- +%% Helper fns +%%--------------------------------------------------------------------------------------- + +do_request(Opts) -> + #{ + path := Path, + method := Method, + headers := Headers, + body := Body0 + } = Opts, + URL = ?HOST ++ filename:join(Path), + Request = + case Body0 of + no_body -> {URL, Headers}; + {Encoding, Body} -> {URL, Headers, Encoding, Body} + end, + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], []) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{_, StatusCode, _}, Headers1, Body1}} -> + Body2 = + case emqx_json:safe_decode(Body1, [return_maps]) of + {ok, Json} -> Json; + {error, _} -> Body1 + end, + {ok, #{status_code => StatusCode, headers => Headers1, body => Body2}} + end. + +%%--------------------------------------------------------------------------------------- +%% Test cases +%%--------------------------------------------------------------------------------------- + +t_status_ok(_Config) -> + {ok, #{ + body := Resp, + status_code := StatusCode + }} = do_request(#{ + method => get, + path => ["status"], + headers => [], + body => no_body + }), + ?assertEqual(200, StatusCode), + ?assertMatch( + {match, _}, + re:run(Resp, <<"emqx is running$">>) + ), + ok. + +t_status_not_ok(_Config) -> + {ok, #{ + body := Resp, + status_code := StatusCode + }} = do_request(#{ + method => get, + path => ["status"], + headers => [], + body => no_body + }), + ?assertEqual(503, StatusCode), + ?assertMatch( + {match, _}, + re:run(Resp, <<"emqx is not_running$">>) + ), ok. diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index a822817d4..52400b5e6 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -9,3 +9,5 @@ ## Bug fixes - Fix error log message when `mechanism` is missing in authentication config [#8924](https://github.com/emqx/emqx/pull/8924). + +- Fixed the response status code for the `/status` endpoint. Before the fix, it always returned 200 even if the EMQX application was not running. Now it returns 503 in that case. diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 0368a6f60..14bb94d1d 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -9,3 +9,5 @@ ## Bug fixes - 优化认认证配置中 `mechanism` 字段缺失情况下的错误日志 [#8924](https://github.com/emqx/emqx/pull/8924)。 + +- 修正了`/status`端点的响应状态代码。 在修复之前,它总是返回200,即使EMQX应用程序没有运行。 现在它在这种情况下返回503。 From 6710736243f1b824d71abab32ec7d4fbe6b61a5b Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 25 Oct 2022 09:41:58 +0800 Subject: [PATCH 081/165] feat: add prometheus collectors config --- apps/emqx_conf/src/emqx_conf_schema.erl | 34 ++++++++++- .../i18n/emqx_prometheus_schema_i18n.conf | 37 ++++++++++++ .../src/emqx_prometheus_schema.erl | 60 +++++++++++++++++++ .../test/emqx_prometheus_SUITE.erl | 9 ++- 4 files changed, 137 insertions(+), 3 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 4de952d2e..8c8bd9152 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -980,7 +980,7 @@ desc("authorization") -> desc(_) -> undefined. -translations() -> ["ekka", "kernel", "emqx", "gen_rpc"]. +translations() -> ["ekka", "kernel", "emqx", "gen_rpc", "prometheus"]. translation("ekka") -> [{"cluster_discovery", fun tr_cluster_discovery/1}]; @@ -997,7 +997,37 @@ translation("emqx") -> {"local_override_conf_file", fun tr_local_override_conf_file/1} ]; translation("gen_rpc") -> - [{"default_client_driver", fun tr_default_config_driver/1}]. + [{"default_client_driver", fun tr_default_config_driver/1}]; +translation("prometheus") -> + [ + {"vm_dist_collector_metrics", fun tr_vm_dist_collector/1}, + {"mnesia_collector_metrics", fun tr_mnesia_collector/1}, + {"vm_statistics_collector_metrics", fun tr_vm_statistics_collector/1}, + {"vm_system_info_collector_metrics", fun tr_vm_system_info_collector/1}, + {"vm_memory_collector_metrics", fun tr_vm_memory_collector/1}, + {"vm_msacc_collector_metrics", fun tr_vm_msacc_collector/1} + ]. + +tr_vm_dist_collector(Conf) -> + metrics_enabled(conf_get("prometheus.vm_dist_collector", Conf, enabled)). + +tr_mnesia_collector(Conf) -> + metrics_enabled(conf_get("prometheus.mnesia_collector", Conf, enabled)). + +tr_vm_statistics_collector(Conf) -> + metrics_enabled(conf_get("prometheus.vm_statistics_collector", Conf, enabled)). + +tr_vm_system_info_collector(Conf) -> + metrics_enabled(conf_get("prometheus.vm_system_info_collector", Conf, enabled)). + +tr_vm_memory_collector(Conf) -> + metrics_enabled(conf_get("prometheus.vm_system_info_collector", Conf, enabled)). + +tr_vm_msacc_collector(Conf) -> + metrics_enabled(conf_get("prometheus.vm_system_info_collector", Conf, enabled)). + +metrics_enabled(enabled) -> all; +metrics_enabled(disabled) -> []. tr_default_config_driver(Conf) -> conf_get("rpc.driver", Conf). diff --git a/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf b/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf index 2c7736938..7f251ff4b 100644 --- a/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf +++ b/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf @@ -30,4 +30,41 @@ emqx_prometheus_schema { zh: """开启或关闭 Prometheus 数据推送""" } } + vm_dist_collector { + desc { + en: """Enable or disable VM distribution collector, collects information about the sockets and processes involved in the Erlang distribution mechanism.""" + zh: """开启或关闭 VM 分布采集器,收集 Erlang 分布机制中涉及的套接字和进程的信息。""" + } + } + mnesia_collector { + desc { + en: """Enable or disable Mnesia collector, collects Mnesia metrics mainly using mnesia:system_info/1 .""" + zh: """开启或关闭 Mnesia 采集器, 使用 mnesia:system_info/1 收集 Mnesia 相关指标""" + } + } + vm_statistics_collector { + desc { + en: """Enable or disable VM statistics collector, collects Erlang VM metrics using erlang:statistics/1 .""" + zh: """开启或关闭 VM 统计采集器, 使用 erlang:statistics/1 收集 Erlang VM 相关指标""" + } + } + + vm_system_info_collector { + desc { + en: """Enable or disable VM system info collector, collects Erlang VM metrics using erlang:system_info/1 .""" + zh: """开启或关闭 VM 系统信息采集器, 使用 erlang:system_info/1 收集 Erlang VM 相关指标""" + } + } + vm_memory_collector { + desc { + en: """Enable or disable VM memory collector, collects information about memory dynamically allocated by the Erlang emulator using erlang:memory/0 , also provides basic (D)ETS statistics .""" + zh: """开启或关闭 VM 内存采集器, 使用 erlang:memory/0 收集 Erlang 虚拟机动态分配的内存信息,同时提供基本的 (D)ETS 统计信息""" + } + } + vm_msacc_collector { + desc { + en: """Enable or disable VM msacc collector, collects microstate accounting metrics using erlang:statistics(microstate_accounting) .""" + zh: """开启或关闭 VM msacc 采集器, 使用 erlang:statistics(microstate_accounting) 收集微状态计数指标""" + } + } } diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index efd453135..4149485ff 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -59,6 +59,66 @@ fields("prometheus") -> required => true, desc => ?DESC(enable) } + )}, + {vm_dist_collector, + ?HOCON( + hoconsc:enum([enabled, disabled]), + #{ + default => enabled, + required => true, + hidden => true, + desc => ?DESC(vm_dist_collector) + } + )}, + {mnesia_collector, + ?HOCON( + hoconsc:enum([enabled, disabled]), + #{ + default => enabled, + required => true, + hidden => true, + desc => ?DESC(mnesia_collector) + } + )}, + {vm_statistics_collector, + ?HOCON( + hoconsc:enum([enabled, disabled]), + #{ + default => enabled, + required => true, + hidden => true, + desc => ?DESC(vm_statistics_collector) + } + )}, + {vm_system_info_collector, + ?HOCON( + hoconsc:enum([enabled, disabled]), + #{ + default => enabled, + required => true, + hidden => true, + desc => ?DESC(vm_system_info_collector) + } + )}, + {vm_memory_collector, + ?HOCON( + hoconsc:enum([enabled, disabled]), + #{ + default => enabled, + required => true, + hidden => true, + desc => ?DESC(vm_memory_collector) + } + )}, + {vm_msacc_collector, + ?HOCON( + hoconsc:enum([enabled, disabled]), + #{ + default => enabled, + required => true, + hidden => true, + desc => ?DESC(vm_msacc_collector) + } )} ]. diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl index 16590b114..3ae21511d 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl @@ -28,6 +28,12 @@ " push_gateway_server = \"http://127.0.0.1:9091\"\n" " interval = \"1s\"\n" " enable = true\n" + " vm_dist_collector = enabled\n" + " mnesia_collector = enabled\n" + " vm_statistics_collector = disabled\n" + " vm_system_info_collector = disabled\n" + " vm_memory_collector = enabled\n" + " vm_msacc_collector = enabled\n" "}\n" >>). @@ -71,7 +77,8 @@ t_start_stop(_) -> %% wait the interval timer tigger timer:sleep(2000). -t_test(_) -> +t_collector_no_crash_test(_) -> + prometheus_text_format:format(), ok. t_only_for_coverage(_) -> From 8578024637bd49e2a0aa465e600cd3c7bd397e53 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 25 Oct 2022 11:21:16 +0800 Subject: [PATCH 082/165] chore: bump emqx_prometheus to 5.0.3 --- apps/emqx_prometheus/src/emqx_prometheus.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index e446a572a..d95c89c3b 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.2"}, + {vsn, "5.0.3"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx]}, From 4c5e64d5a3df41edf0471becf472b2d9c5024f8f Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 25 Oct 2022 12:27:25 +0800 Subject: [PATCH 083/165] feat: hibernate after send data to prometheus push_gateway --- apps/emqx_prometheus/src/emqx_prometheus.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 4bbfbe524..d2c09774c 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -147,7 +147,8 @@ handle_info({timeout, R, ?TIMER_MSG}, State = #state{timer = R, push_gateway = U Url = lists:concat([Uri, "/metrics/job/", Name, "/instance/", Name, "~", Ip]), Data = prometheus_text_format:format(), httpc:request(post, {Url, [], "text/plain", Data}, [{autoredirect, true}], []), - {noreply, ensure_timer(State)}; + %% Data is too big, hibernate for saving memory and stop system monitor warning. + {noreply, ensure_timer(State), hibernate}; handle_info(_Msg, State) -> {noreply, State}. From 049f988642231a4918a31134e36fa0f7bb4ed36c Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 25 Oct 2022 22:07:48 +0800 Subject: [PATCH 084/165] chore: update apps/emqx_conf/src/emqx_conf_schema.erl Co-authored-by: Thales Macedo Garitezi --- apps/emqx_conf/src/emqx_conf_schema.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 8c8bd9152..a68c11dc4 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1021,10 +1021,10 @@ tr_vm_system_info_collector(Conf) -> metrics_enabled(conf_get("prometheus.vm_system_info_collector", Conf, enabled)). tr_vm_memory_collector(Conf) -> - metrics_enabled(conf_get("prometheus.vm_system_info_collector", Conf, enabled)). + metrics_enabled(conf_get("prometheus.vm_memory_collector", Conf, enabled)). tr_vm_msacc_collector(Conf) -> - metrics_enabled(conf_get("prometheus.vm_system_info_collector", Conf, enabled)). + metrics_enabled(conf_get("prometheus.vm_msacc_collector", Conf, enabled)). metrics_enabled(enabled) -> all; metrics_enabled(disabled) -> []. From 156b19b525e2bd57942dfd5ca3e43081cbaf52e0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 25 Oct 2022 17:24:36 +0200 Subject: [PATCH 085/165] docs: update v5.0.10-zh.md --- changes/v5.0.10-zh.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 14bb94d1d..3288efcda 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -10,4 +10,5 @@ - 优化认认证配置中 `mechanism` 字段缺失情况下的错误日志 [#8924](https://github.com/emqx/emqx/pull/8924)。 -- 修正了`/status`端点的响应状态代码。 在修复之前,它总是返回200,即使EMQX应用程序没有运行。 现在它在这种情况下返回503。 +- 修正了 `/status` 端点的响应状态代码 [#9211](https://github.com/emqx/emqx/pull/9211)。 + 在此修复前,它总是返回 HTTP 状态码 `200`,即使 EMQX 没有完成启动或正在重启。 现在它在这些情况下会返回状态码 `503`。 From 79f2d3e9c3e4578225ef93f27aa5752f853b9e9b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 25 Oct 2022 17:26:05 +0200 Subject: [PATCH 086/165] docs: update v5.0.10-en.md --- changes/v5.0.10-en.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index 52400b5e6..b63cd6e09 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -10,4 +10,5 @@ - Fix error log message when `mechanism` is missing in authentication config [#8924](https://github.com/emqx/emqx/pull/8924). -- Fixed the response status code for the `/status` endpoint. Before the fix, it always returned 200 even if the EMQX application was not running. Now it returns 503 in that case. +- Fixed the HTTP response status code for the `/status` endpoint [#9211](https://github.com/emqx/emqx/pull/9211). + Before the fix, it always returned `200` even if the EMQX application was not running. Now it returns `503` in that case. From 0eca531e64b22d7ad16ce9b38eb59d2d8ed93b07 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 25 Oct 2022 15:01:05 -0300 Subject: [PATCH 087/165] feat: add `retry-after` headers to unavailable response --- .../src/emqx_mgmt_api_status.erl | 6 +- .../test/emqx_mgmt_api_status_SUITE.erl | 60 ++++++++++++------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_status.erl b/apps/emqx_management/src/emqx_mgmt_api_status.erl index 37a8274f8..b313f4bcc 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_status.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_status.erl @@ -50,7 +50,11 @@ running_status() -> running -> 200; not_running -> 503 end, - {StatusCode, #{<<"content-type">> => <<"text/plain">>}, list_to_binary(Body)}; + Headers = #{ + <<"content-type">> => <<"text/plain">>, + <<"retry-after">> => <<"15">> + }, + {StatusCode, Headers, list_to_binary(Body)}; false -> {503, #{<<"retry-after">> => <<"15">>}, <<>>} end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl index 4051bd62c..42d833bda 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl @@ -54,28 +54,43 @@ end_per_testcase(_TestCase, _Config) -> do_request(Opts) -> #{ - path := Path, + path := Path0, method := Method, headers := Headers, body := Body0 } = Opts, - URL = ?HOST ++ filename:join(Path), + URL = ?HOST ++ filename:join(Path0), + {ok, #{host := Host, port := Port, path := Path}} = emqx_http_lib:uri_parse(URL), + %% we must not use `httpc' here, because it keeps retrying when it + %% receives a 503 with `retry-after' header, and there's no option + %% to stop that behavior... + {ok, Gun} = gun:open(Host, Port, #{retry => 0}), + {ok, http} = gun:await_up(Gun), Request = - case Body0 of - no_body -> {URL, Headers}; - {Encoding, Body} -> {URL, Headers, Encoding, Body} + fun() -> + case Body0 of + no_body -> gun:Method(Gun, Path, Headers); + {_Encoding, Body} -> gun:Method(Gun, Path, Headers, Body) + end end, - ct:pal("Method: ~p, Request: ~p", [Method, Request]), - case httpc:request(Method, Request, [], []) of - {error, socket_closed_remotely} -> - {error, socket_closed_remotely}; - {ok, {{_, StatusCode, _}, Headers1, Body1}} -> - Body2 = - case emqx_json:safe_decode(Body1, [return_maps]) of - {ok, Json} -> Json; - {error, _} -> Body1 - end, - {ok, #{status_code => StatusCode, headers => Headers1, body => Body2}} + Ref = Request(), + receive + {gun_response, Gun, Ref, nofin, StatusCode, Headers1} -> + Data = data_loop(Gun, Ref, _Acc = <<>>), + #{status_code => StatusCode, headers => maps:from_list(Headers1), body => Data} + after 5_000 -> + error({timeout, Opts, process_info(self(), messages)}) + end. + +data_loop(Gun, Ref, Acc) -> + receive + {gun_data, Gun, Ref, nofin, Data} -> + data_loop(Gun, Ref, <>); + {gun_data, Gun, Ref, fin, Data} -> + gun:shutdown(Gun), + <> + after 5000 -> + error(timeout) end. %%--------------------------------------------------------------------------------------- @@ -83,10 +98,10 @@ do_request(Opts) -> %%--------------------------------------------------------------------------------------- t_status_ok(_Config) -> - {ok, #{ + #{ body := Resp, status_code := StatusCode - }} = do_request(#{ + } = do_request(#{ method => get, path => ["status"], headers => [], @@ -100,10 +115,11 @@ t_status_ok(_Config) -> ok. t_status_not_ok(_Config) -> - {ok, #{ + #{ body := Resp, + headers := Headers, status_code := StatusCode - }} = do_request(#{ + } = do_request(#{ method => get, path => ["status"], headers => [], @@ -114,4 +130,8 @@ t_status_not_ok(_Config) -> {match, _}, re:run(Resp, <<"emqx is not_running$">>) ), + ?assertMatch( + #{<<"retry-after">> := <<"15">>}, + Headers + ), ok. From 13543c45cc5d32dc501affea3ae2a1a537eaa714 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 26 Oct 2022 10:04:28 +0200 Subject: [PATCH 088/165] chore: add pull request template --- .github/PULL_REQUEST_TEMPLATE.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e69de29bb..41c6276a3 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,18 @@ + +Fixes + +**If your build fails** due to your commit message not passing the build checks, please review the guidelines here: https://github.com/emqx/emqx/blob/master/CONTRIBUTING.md. + +## PR Checklist +Please convert it to a draft if any of the following conditions are not met. Reviewers may skip over until all the items are checked: + +- [ ] Added tests for the changes +- [ ] Changed lines covered in coverage report +- [ ] Change log has been added to `changes/` dir +- [ ] For EMQX 4.x: `appup` files updated (execute `scripts/update-appup.sh emqx`) +- [ ] For internal contributor: there is a jira ticket to track this change, and another jira tickt to track doc updates (if any) +- [ ] In case of non-backward compatible changes, reviewer should check this item as a write-off, and add details in **Backward Compatibility** section + +## Backward Compatibility + +## More information From 6c8936467ea06fbb4500b2d9b7ad16678f1b7d1a Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 26 Oct 2022 11:59:37 +0200 Subject: [PATCH 089/165] chore(ekka): Bump version to 0.13.6 --- apps/emqx/rebar.config | 2 +- changes/v5.0.10-en.md | 2 ++ changes/v5.0.10-zh.md | 2 ++ mix.exs | 2 +- rebar.config | 2 +- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index d20e980ea..156ca5c21 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.5"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.6"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.30.0"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index a822817d4..2cab5e449 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -6,6 +6,8 @@ - Update `gen_rpc` library to version 3.0 [#9187](https://github.com/emqx/emqx/pull/9187). +- Improve memory usage on core nodes when bootstrapping a replicant [9236](https://github.com/emqx/emqx/pull/9236). + ## Bug fixes - Fix error log message when `mechanism` is missing in authentication config [#8924](https://github.com/emqx/emqx/pull/8924). diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 0368a6f60..8ca3be883 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -6,6 +6,8 @@ - 升级 `gen_rpc` 库到 3.0 [#9187](https://github.com/emqx/emqx/pull/9187)。 +- 在引导 `replicant`"时,改善 `core` "节点的内存使用 [9236](https://github.com/emqx/emqx/pull/9236)。 + ## Bug fixes - 优化认认证配置中 `mechanism` 字段缺失情况下的错误日志 [#8924](https://github.com/emqx/emqx/pull/8924)。 diff --git a/mix.exs b/mix.exs index 43024c116..de827bc07 100644 --- a/mix.exs +++ b/mix.exs @@ -52,7 +52,7 @@ defmodule EMQXUmbrella.MixProject do {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.4", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.13.5", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.13.6", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.7", override: true}, diff --git a/rebar.config b/rebar.config index f415b5901..80ae099ac 100644 --- a/rebar.config +++ b/rebar.config @@ -54,7 +54,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.5"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.6"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.7"}}} From 702c968f4646e6ee5bbdc1c52a3eb3613bcfed34 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 26 Oct 2022 13:02:35 +0200 Subject: [PATCH 090/165] chore(ekka): Apply suggestions from code review Co-authored-by: Zaiming (Stone) Shi --- changes/v5.0.10-en.md | 2 +- changes/v5.0.10-zh.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index 2cab5e449..c70ec9977 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -6,7 +6,7 @@ - Update `gen_rpc` library to version 3.0 [#9187](https://github.com/emqx/emqx/pull/9187). -- Improve memory usage on core nodes when bootstrapping a replicant [9236](https://github.com/emqx/emqx/pull/9236). +- Improve memory usage on core nodes when bootstrapping a replicant [#9236](https://github.com/emqx/emqx/pull/9236). ## Bug fixes diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 8ca3be883..26b2737dd 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -6,7 +6,7 @@ - 升级 `gen_rpc` 库到 3.0 [#9187](https://github.com/emqx/emqx/pull/9187)。 -- 在引导 `replicant`"时,改善 `core` "节点的内存使用 [9236](https://github.com/emqx/emqx/pull/9236)。 +- 在引导 `replicant` 节点时,改善 `core` 节点的内存使用量 [#9236](https://github.com/emqx/emqx/pull/9236)。 ## Bug fixes From e22f8ff2a84d937ded206c5346797c637b4f545b Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 26 Oct 2022 17:06:32 +0800 Subject: [PATCH 091/165] refactor: sync emqx_prometheus via emqx_config_handler --- .../include/emqx_prometheus.hrl | 1 + apps/emqx_prometheus/src/emqx_prometheus.erl | 112 +++++++----------- .../src/emqx_prometheus_api.erl | 10 +- .../src/emqx_prometheus_app.erl | 15 +-- .../src/emqx_prometheus_config.erl | 51 ++++++++ .../src/emqx_prometheus_schema.erl | 9 +- .../src/emqx_prometheus_sup.erl | 21 ++-- .../test/emqx_prometheus_SUITE.erl | 2 +- .../test/emqx_prometheus_api_SUITE.erl | 17 ++- 9 files changed, 138 insertions(+), 100 deletions(-) create mode 100644 apps/emqx_prometheus/src/emqx_prometheus_config.erl diff --git a/apps/emqx_prometheus/include/emqx_prometheus.hrl b/apps/emqx_prometheus/include/emqx_prometheus.hrl index 589bbd024..36066a55d 100644 --- a/apps/emqx_prometheus/include/emqx_prometheus.hrl +++ b/apps/emqx_prometheus/include/emqx_prometheus.hrl @@ -1 +1,2 @@ -define(APP, emqx_prometheus). +-define(PROMETHEUS, [prometheus]). diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index d2c09774c..4edf371cc 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -37,18 +37,8 @@ ] ). --export([ - update/1, - start/0, - stop/0, - restart/0, - % for rpc - do_start/0, - do_stop/0 -]). - %% APIs --export([start_link/1]). +-export([start_link/1, info/0]). %% gen_server callbacks -export([ @@ -73,84 +63,59 @@ -define(TIMER_MSG, '#interval'). --record(state, {push_gateway, timer, interval}). - -%%-------------------------------------------------------------------- -%% update new config -update(Config) -> - case - emqx_conf:update( - [prometheus], - Config, - #{rawconf_with_defaults => true, override_to => cluster} - ) - of - {ok, #{raw_config := NewConfigRows}} -> - case maps:get(<<"enable">>, Config, true) of - true -> - ok = restart(); - false -> - ok = stop() - end, - {ok, NewConfigRows}; - {error, Reason} -> - {error, Reason} - end. - -start() -> - {_, []} = emqx_prometheus_proto_v1:start(mria_mnesia:running_nodes()), - ok. - -stop() -> - {_, []} = emqx_prometheus_proto_v1:stop(mria_mnesia:running_nodes()), - ok. - -restart() -> - ok = stop(), - ok = start(). - -do_start() -> - emqx_prometheus_sup:start_child(?APP, emqx_conf:get([prometheus])). - -do_stop() -> - case emqx_prometheus_sup:stop_child(?APP) of - ok -> - ok; - {error, not_found} -> - ok - end. +-define(HTTP_OPTIONS, [{autoredirect, true}, {timeout, 60000}]). %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- -start_link(Opts) -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [Opts], []). +start_link([]) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +info() -> + gen_server:call(?MODULE, info). %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- -init([Opts]) -> - Interval = maps:get(interval, Opts), - PushGateway = maps:get(push_gateway_server, Opts), - {ok, ensure_timer(#state{push_gateway = PushGateway, interval = Interval})}. +init([]) -> + #{interval := Interval} = opts(), + {ok, #{timer => ensure_timer(Interval), ok => 0, failed => 0}}. +handle_call(info, _From, State) -> + {reply, State#{opts => opts()}, State}; handle_call(_Msg, _From, State) -> - {noreply, State}. + {reply, ok, State}. handle_cast(_Msg, State) -> {noreply, State}. -handle_info({timeout, R, ?TIMER_MSG}, State = #state{timer = R, push_gateway = Uri}) -> +handle_info({timeout, Timer, ?TIMER_MSG}, State = #{timer := Timer}) -> + #{interval := Interval, push_gateway_server := Server} = opts(), + PushRes = push_to_push_gateway(Server), + NewTimer = ensure_timer(Interval), + NewState = maps:update_with(PushRes, fun(C) -> C + 1 end, 1, State#{timer => NewTimer}), + %% Data is too big, hibernate for saving memory and stop system monitor warning. + {noreply, NewState, hibernate}; +handle_info(_Msg, State) -> + {noreply, State}. + +push_to_push_gateway(Uri) -> [Name, Ip] = string:tokens(atom_to_list(node()), "@"), Url = lists:concat([Uri, "/metrics/job/", Name, "/instance/", Name, "~", Ip]), Data = prometheus_text_format:format(), - httpc:request(post, {Url, [], "text/plain", Data}, [{autoredirect, true}], []), - %% Data is too big, hibernate for saving memory and stop system monitor warning. - {noreply, ensure_timer(State), hibernate}; -handle_info(_Msg, State) -> - {noreply, State}. + case httpc:request(post, {Url, [], "text/plain", Data}, ?HTTP_OPTIONS, []) of + {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, _Body}} -> + ok; + Error -> + ?SLOG(error, #{ + msg => "post_to_push_gateway_failed", + error => Error, + url => Url + }), + failed + end. code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -158,11 +123,14 @@ code_change(_OldVsn, State, _Extra) -> terminate(_Reason, _State) -> ok. -ensure_timer(State = #state{interval = Interval}) -> - State#state{timer = emqx_misc:start_timer(Interval, ?TIMER_MSG)}. +ensure_timer(Interval) -> + emqx_misc:start_timer(Interval, ?TIMER_MSG). + %%-------------------------------------------------------------------- %% prometheus callbacks %%-------------------------------------------------------------------- +opts() -> + emqx_conf:get(?PROMETHEUS). deregister_cleanup(_Registry) -> ok. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index 9a81f3ea3..125eed560 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -84,7 +84,7 @@ schema("/prometheus/stats") -> prometheus(get, _Params) -> {200, emqx:get_raw_config([<<"prometheus">>], #{})}; prometheus(put, #{body := Body}) -> - case emqx_prometheus:update(Body) of + case emqx_prometheus_config:update(Body) of {ok, NewConfig} -> {200, NewConfig}; {error, Reason} -> @@ -120,7 +120,13 @@ prometheus_config_example() -> #{ enable => true, interval => "15s", - push_gateway_server => <<"http://127.0.0.1:9091">> + push_gateway_server => <<"http://127.0.0.1:9091">>, + vm_dist_collector => enabled, + mnesia_collector => enabled, + vm_statistics_collector => enabled, + vm_system_info_collector => enabled, + vm_memory_collector => enabled, + vm_msacc_collector => enabled }. prometheus_data_schema() -> diff --git a/apps/emqx_prometheus/src/emqx_prometheus_app.erl b/apps/emqx_prometheus/src/emqx_prometheus_app.erl index b9dd9c466..bdee12d0e 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_app.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_app.erl @@ -27,17 +27,10 @@ ]). start(_StartType, _StartArgs) -> - {ok, Sup} = emqx_prometheus_sup:start_link(), - maybe_enable_prometheus(), - {ok, Sup}. + Res = emqx_prometheus_sup:start_link(), + emqx_prometheus_config:add_handler(), + Res. stop(_State) -> + emqx_prometheus_config:remove_handler(), ok. - -maybe_enable_prometheus() -> - case emqx_conf:get([prometheus, enable], false) of - true -> - emqx_prometheus_sup:start_child(?APP, emqx_conf:get([prometheus], #{})); - false -> - ok - end. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_config.erl b/apps/emqx_prometheus/src/emqx_prometheus_config.erl new file mode 100644 index 000000000..83762b7c7 --- /dev/null +++ b/apps/emqx_prometheus/src/emqx_prometheus_config.erl @@ -0,0 +1,51 @@ +%%%------------------------------------------------------------------- +%%% @author zhongwen +%%% @copyright (C) 2022, +%%% @doc +%%% +%%% @end +%%% Created : 26. 10月 2022 11:14 +%%%------------------------------------------------------------------- +-module(emqx_prometheus_config). + +-behaviour(emqx_config_handler). + +-include("emqx_prometheus.hrl"). + +-export([add_handler/0, remove_handler/0]). +-export([post_config_update/5]). +-export([update/1]). + +update(Config) -> + case + emqx_conf:update( + [prometheus], + Config, + #{rawconf_with_defaults => true, override_to => cluster} + ) + of + {ok, #{raw_config := NewConfigRows}} -> + {ok, NewConfigRows}; + {error, Reason} -> + {error, Reason} + end. + +add_handler() -> + ok = emqx_config_handler:add_handler(?PROMETHEUS, ?MODULE), + ok. + +remove_handler() -> + ok = emqx_config_handler:remove_handler(?PROMETHEUS), + ok. + +post_config_update(?PROMETHEUS, _Req, New, _Old, AppEnvs) -> + application:set_env(AppEnvs), + update_prometheus(New), + ok; +post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> + ok. + +update_prometheus(#{enable := true}) -> + emqx_prometheus_sup:start_child(?APP); +update_prometheus(#{enable := false}) -> + emqx_prometheus_sup:stop_child(?APP). diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 4149485ff..09908167c 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -24,12 +24,13 @@ namespace/0, roots/0, fields/1, - desc/1 + desc/1, + translation/1 ]). namespace() -> "prometheus". -roots() -> ["prometheus"]. +roots() -> [{"prometheus", ?HOCON(?R_REF("prometheus"), #{translate_to => ["prometheus"]})}]. fields("prometheus") -> [ @@ -124,3 +125,7 @@ fields("prometheus") -> desc("prometheus") -> ?DESC(prometheus); desc(_) -> undefined. + +%% for CI test, CI don't load the whole emqx_conf_schema. +translation(Name) -> + emqx_conf_schema:translation(Name). diff --git a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl index 65023da14..eaf96af43 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl @@ -21,7 +21,6 @@ -export([ start_link/0, start_child/1, - start_child/2, stop_child/1 ]). @@ -40,23 +39,27 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). --spec start_child(supervisor:child_spec()) -> ok. +-spec start_child(supervisor:child_spec() | atom()) -> ok. start_child(ChildSpec) when is_map(ChildSpec) -> - assert_started(supervisor:start_child(?MODULE, ChildSpec)). - --spec start_child(atom(), map()) -> ok. -start_child(Mod, Opts) when is_atom(Mod) andalso is_map(Opts) -> - assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Opts))). + assert_started(supervisor:start_child(?MODULE, ChildSpec)); +start_child(Mod) when is_atom(Mod) -> + assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, []))). -spec stop_child(any()) -> ok | {error, term()}. stop_child(ChildId) -> case supervisor:terminate_child(?MODULE, ChildId) of ok -> supervisor:delete_child(?MODULE, ChildId); + {error, not_found} -> ok; Error -> Error end. init([]) -> - {ok, {{one_for_one, 10, 3600}, []}}. + Children = + case emqx_conf:get([prometheus, enable], false) of + false -> []; + true -> [?CHILD(emqx_prometheus, [])] + end, + {ok, {{one_for_one, 10, 3600}, Children}}. %%-------------------------------------------------------------------- %% Internal functions @@ -64,5 +67,5 @@ init([]) -> assert_started({ok, _Pid}) -> ok; assert_started({ok, _Pid, _Info}) -> ok; -assert_started({error, {already_tarted, _Pid}}) -> ok; +assert_started({error, {already_started, _Pid}}) -> ok; assert_started({error, Reason}) -> erlang:error(Reason). diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl index 3ae21511d..1a72b8952 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl @@ -74,7 +74,7 @@ t_start_stop(_) -> ?assertMatch(ok, emqx_prometheus:start()), ?assertMatch(ok, emqx_prometheus:stop()), ?assertMatch(ok, emqx_prometheus:restart()), - %% wait the interval timer tigger + %% wait the interval timer trigger timer:sleep(2000). t_collector_no_crash_test(_) -> diff --git a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl index e72d7865a..59b3b9a17 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl @@ -71,16 +71,27 @@ t_prometheus_api(_) -> #{ <<"push_gateway_server">> := _, <<"interval">> := _, - <<"enable">> := _ + <<"enable">> := _, + <<"vm_statistics_collector">> := _, + <<"vm_system_info_collector">> := _, + <<"vm_memory_collector">> := _, + <<"vm_msacc_collector">> := _ }, Conf ), - - NewConf = Conf#{<<"interval">> := <<"2s">>}, + #{<<"enable">> := Enable} = Conf, + ?assertEqual(Enable, undefined =/= erlang:whereis(emqx_prometheus)), + NewConf = Conf#{<<"interval">> => <<"2s">>, <<"vm_statistics_collector">> => <<"disabled">>}, {ok, Response2} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf), Conf2 = emqx_json:decode(Response2, [return_maps]), ?assertMatch(NewConf, Conf2), + ?assertEqual({ok, []}, application:get_env(prometheus, vm_statistics_collector_metrics)), + ?assertEqual({ok, all}, application:get_env(prometheus, vm_memory_collector_metrics)), + + NewConf1 = Conf#{<<"enable">> => (not Enable)}, + {ok, _Response3} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf1), + ?assertEqual((not Enable), undefined =/= erlang:whereis(emqx_prometheus)), ok. t_stats_api(_) -> From a3f877656e23a29a13d229c6a09447a906c4472b Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 26 Oct 2022 17:37:40 +0800 Subject: [PATCH 092/165] chore: add changlog for emqx_prometheus's change --- changes/v5.0.10-en.md | 4 ++++ changes/v5.0.10-zh.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index c70ec9977..2b49aebf0 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -8,6 +8,10 @@ - Improve memory usage on core nodes when bootstrapping a replicant [#9236](https://github.com/emqx/emqx/pull/9236). +- Improve the stability of prometheus push_gateway and print error logs when POST fails [#9235](http://github.com/emqx/emqx/pull/9235). + +- Now we can disable some internal prometheus metrics. Closing some metrics that are not cared about when the prometheus/stats API timeout ( the machine load is too high ). [#9222](https://github.com/emqx/emqx/pull/9222). + ## Bug fixes - Fix error log message when `mechanism` is missing in authentication config [#8924](https://github.com/emqx/emqx/pull/8924). diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 26b2737dd..636e7fbd6 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -8,6 +8,10 @@ - 在引导 `replicant` 节点时,改善 `core` 节点的内存使用量 [#9236](https://github.com/emqx/emqx/pull/9236)。 +- 增加 prometheus push_gateway 的稳定性, 并在 POST 失败时打印错误日志 [#9235](http://github.com/emqx/emqx/pull/9235)。 + +- 可通过配置关闭 prometheus 中的部分内部指标,如果遇到机器负载过高 prometheus 接口返回超时可考虑关闭部分不关心指标,以提高响应速度 [#9222](https://github.com/emqx/emqx/pull/9222)。 + ## Bug fixes - 优化认认证配置中 `mechanism` 字段缺失情况下的错误日志 [#8924](https://github.com/emqx/emqx/pull/8924)。 From 1757342e5ba5c896359eeff4b27539640d99967b Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 26 Oct 2022 17:51:40 +0800 Subject: [PATCH 093/165] chore: make dialyzer happy --- .../src/emqx_prometheus_config.erl | 26 ++++++++++++------- .../src/emqx_prometheus_sup.erl | 2 +- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus_config.erl b/apps/emqx_prometheus/src/emqx_prometheus_config.erl index 83762b7c7..b4914f216 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_config.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_config.erl @@ -1,11 +1,18 @@ -%%%------------------------------------------------------------------- -%%% @author zhongwen -%%% @copyright (C) 2022, -%%% @doc -%%% -%%% @end -%%% Created : 26. 10月 2022 11:14 -%%%------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_prometheus_config). -behaviour(emqx_config_handler). @@ -40,8 +47,7 @@ remove_handler() -> post_config_update(?PROMETHEUS, _Req, New, _Old, AppEnvs) -> application:set_env(AppEnvs), - update_prometheus(New), - ok; + update_prometheus(New); post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> ok. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl index eaf96af43..a70fda322 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl @@ -68,4 +68,4 @@ init([]) -> assert_started({ok, _Pid}) -> ok; assert_started({ok, _Pid, _Info}) -> ok; assert_started({error, {already_started, _Pid}}) -> ok; -assert_started({error, Reason}) -> erlang:error(Reason). +assert_started({error, Reason}) -> {error, Reason}. From 97124be63afe361287714486c938e8eba926a503 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 26 Oct 2022 18:03:01 +0800 Subject: [PATCH 094/165] chore: add next_push_ms to emqx_prometheus:info/0 --- apps/emqx_prometheus/src/emqx_prometheus.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 4edf371cc..32f9928fb 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -83,8 +83,8 @@ init([]) -> #{interval := Interval} = opts(), {ok, #{timer => ensure_timer(Interval), ok => 0, failed => 0}}. -handle_call(info, _From, State) -> - {reply, State#{opts => opts()}, State}; +handle_call(info, _From, State = #{timer := Timer}) -> + {reply, State#{opts => opts(), next_push_ms => erlang:read_timer(Timer)}, State}; handle_call(_Msg, _From, State) -> {reply, ok, State}. From 3233a40af3289c4ce031984d072f2d39370c538c Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 26 Oct 2022 21:58:51 +0800 Subject: [PATCH 095/165] chore: don't delete bpapi when 5.0.x --- apps/emqx_prometheus/src/emqx_prometheus.erl | 14 ++++++++++++++ .../src/proto/emqx_prometheus_proto_v1.erl | 4 +++- changes/v5.0.10-en.md | 5 +++-- changes/v5.0.10-zh.md | 2 +- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 32f9928fb..de9349b97 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -59,6 +59,12 @@ -export([collect/1]). +-export([ + %% For bpapi, deprecated_since 5.0.10, remove this when 5.1.x + do_start/0, + do_stop/0 +]). + -define(C(K, L), proplists:get_value(K, L, 0)). -define(TIMER_MSG, '#interval'). @@ -591,3 +597,11 @@ emqx_cluster_data() -> {nodes_running, length(Running)}, {nodes_stopped, length(Stopped)} ]. + +%% deprecated_since 5.0.10, remove this when 5.1.x +do_start() -> + emqx_prometheus_sup:start_child(?APP). + +%% deprecated_since 5.0.10, remove this when 5.1.x +do_stop() -> + emqx_prometheus_sup:stop_child(?APP). diff --git a/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl b/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl index c0529cabd..e11f8e3ad 100644 --- a/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl +++ b/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl @@ -20,13 +20,15 @@ -export([ introduced_in/0, - + deprecated_since/0, start/1, stop/1 ]). -include_lib("emqx/include/bpapi.hrl"). +deprecated_since() -> "5.0.10". + introduced_in() -> "5.0.0". diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index 2b49aebf0..4bb7332f9 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -8,9 +8,10 @@ - Improve memory usage on core nodes when bootstrapping a replicant [#9236](https://github.com/emqx/emqx/pull/9236). -- Improve the stability of prometheus push_gateway and print error logs when POST fails [#9235](http://github.com/emqx/emqx/pull/9235). +- Improve stability of Prometheus Push Gateway and log errors when POST fails [#9235](http://github.com/emqx/emqx/pull/9235). -- Now we can disable some internal prometheus metrics. Closing some metrics that are not cared about when the prometheus/stats API timeout ( the machine load is too high ). [#9222](https://github.com/emqx/emqx/pull/9222). +- Now it is possible to opt out VM internal metrics in prometheus stats [#9222](https://github.com/emqx/emqx/pull/9222). + When system load is high, reporting too much metrics data may cause the prometheus stats API timeout. ## Bug fixes diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 636e7fbd6..df5a16eed 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -8,7 +8,7 @@ - 在引导 `replicant` 节点时,改善 `core` 节点的内存使用量 [#9236](https://github.com/emqx/emqx/pull/9236)。 -- 增加 prometheus push_gateway 的稳定性, 并在 POST 失败时打印错误日志 [#9235](http://github.com/emqx/emqx/pull/9235)。 +- 增加 Prometheus Push Gateway 的稳定性, 并在 POST 失败时打印错误日志 [#9235](http://github.com/emqx/emqx/pull/9235)。 - 可通过配置关闭 prometheus 中的部分内部指标,如果遇到机器负载过高 prometheus 接口返回超时可考虑关闭部分不关心指标,以提高响应速度 [#9222](https://github.com/emqx/emqx/pull/9222)。 From d6dfead31d663bee61ea07b00c48b2d617041ac3 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 26 Oct 2022 22:05:20 +0800 Subject: [PATCH 096/165] fix: prometheus ct failed --- apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl index 1a72b8952..e26bcfeb4 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl @@ -71,9 +71,13 @@ load_config() -> %%-------------------------------------------------------------------- t_start_stop(_) -> - ?assertMatch(ok, emqx_prometheus:start()), - ?assertMatch(ok, emqx_prometheus:stop()), - ?assertMatch(ok, emqx_prometheus:restart()), + App = emqx_prometheus, + ?assertMatch(ok, emqx_prometheus_sup:start_child(App)), + %% start twice return ok. + ?assertMatch(ok, emqx_prometheus_sup:start_child(App)), + ?assertMatch(ok, emqx_prometheus_sup:stop_child(App)), + %% stop twice return ok. + ?assertMatch(ok, emqx_prometheus_sup:stop_child(App)), %% wait the interval timer trigger timer:sleep(2000). From a314950be9ca260923d07d3c4a9a8be277634727 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 26 Oct 2022 11:07:27 +0200 Subject: [PATCH 097/165] docs: fix zh punctuations --- apps/emqx/i18n/emqx_limiter_i18n.conf | 2 +- apps/emqx/i18n/emqx_schema_i18n.conf | 375 +++++++----------- apps/emqx_authn/i18n/emqx_authn_jwt_i18n.conf | 2 +- .../i18n/emqx_authz_api_cache_i18n.conf | 2 +- .../i18n/emqx_authz_api_schema_i18n.conf | 4 +- .../i18n/emqx_authz_api_settings_i18n.conf | 4 +- .../i18n/emqx_authz_api_sources_i18n.conf | 20 +- .../i18n/emqx_authz_schema_i18n.conf | 76 ++-- .../i18n/emqx_auto_subscribe_i18n.conf | 2 +- apps/emqx_conf/i18n/emqx_conf_schema.conf | 4 +- .../i18n/emqx_connector_mqtt_schema.conf | 6 +- .../i18n/emqx_gateway_schema_i18n.conf | 60 +-- .../i18n/emqx_plugins_schema.conf | 2 +- apps/emqx_psk/i18n/emqx_psk_i18n.conf | 2 +- .../src/emqx_rule_engine.app.src | 2 +- .../emqx_rule_engine/src/emqx_rule_events.erl | 4 +- rel/emqx_conf.template.en.md | 16 +- rel/emqx_conf.template.zh.md | 45 +-- 18 files changed, 265 insertions(+), 363 deletions(-) diff --git a/apps/emqx/i18n/emqx_limiter_i18n.conf b/apps/emqx/i18n/emqx_limiter_i18n.conf index 99ecc9e1e..b94750b38 100644 --- a/apps/emqx/i18n/emqx_limiter_i18n.conf +++ b/apps/emqx/i18n/emqx_limiter_i18n.conf @@ -171,7 +171,7 @@ Once the limit is reached, the restricted client will be slow down even be hung en: """The bytes_in limiter. This is used to limit the inbound bytes rate for this EMQX node. Once the limit is reached, the restricted client will be slow down even be hung for a while.""" - zh: """流入字节率控制器. + zh: """流入字节率控制器。 这个是用来控制当前节点上的数据流入的字节率,每条消息将会消耗和其二进制大小等量的令牌,当达到最大速率后,会话将会被限速甚至被强制挂起一小段时间""" } label: { diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index e51c953bf..e798b593d 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -3,10 +3,10 @@ emqx_schema { force_shutdown_enable { desc { en: "Enable `force_shutdown` feature." - zh: "启用 `force_shutdown` 功能" + zh: "启用 `force_shutdown` 功能。" } label { - en: "Enable `force_shutdown` feature." + en: "Enable `force_shutdown` feature" zh: "启用 `force_shutdown` 功能" } } @@ -14,7 +14,7 @@ emqx_schema { force_shutdown_max_message_queue_len { desc { en: "Maximum message queue length." - zh: "消息队列的最大长度" + zh: "消息队列的最大长度。" } label { en: "Maximum mailbox queue length of process." @@ -25,7 +25,7 @@ emqx_schema { force_shutdown_max_heap_size { desc { en: "Total heap size" - zh: "Heap 的总大小" + zh: "Heap 的总大小。" } label { en: "Total heap size" @@ -35,8 +35,8 @@ emqx_schema { overload_protection_enable { desc { - en: "React on system overload or not" - zh: "是否对系统过载做出反应" + en: "React on system overload or not." + zh: "是否对系统过载做出反应。" } label { en: "React on system overload or not" @@ -46,9 +46,8 @@ emqx_schema { overload_protection_backoff_delay { desc { - en: "Some unimportant tasks could be delayed " - "for execution, here set the delays in ms" - zh: "一些不重要的任务可能会延迟执行,以毫秒为单位设置延迟" + en: "When at high load, some unimportant tasks could be delayed for execution, here set the duration in milliseconds precision." + zh: "高负载时,一些不重要的任务可能会延迟执行,在这里设置允许延迟的时间。单位为毫秒。" } label { en: "Delay Time" @@ -58,8 +57,8 @@ emqx_schema { overload_protection_backoff_gc { desc { - en: "Skip forceful GC if necessary" - zh: "如有必要,跳过强制GC" + en: "When at high load, skip forceful GC." + zh: "高负载时,跳过强制 GC。" } label { en: "Skip GC" @@ -69,8 +68,8 @@ emqx_schema { overload_protection_backoff_hibernation { desc { - en: "Skip process hibernation if necessary" - zh: "如有必要,跳过进程休眠" + en: "When at high load, skip process hibernation." + zh: "高负载时,跳过进程休眠。" } label { en: "Skip hibernation" @@ -80,8 +79,8 @@ emqx_schema { overload_protection_backoff_new_conn { desc { - en: "Close new incoming connections if necessary" - zh: "如有必要,关闭新进来的连接" + en: "When at high load, close new incoming connections." + zh: "高负载时,拒绝新进来的客户端连接。" } label { en: "Close new connections" @@ -92,7 +91,7 @@ emqx_schema { conn_congestion_enable_alarm { desc { en: "Enable or disable connection congestion alarm." - zh: "启用或者禁用连接阻塞告警功能" + zh: "启用或者禁用连接阻塞告警功能。" } label { en: "Enable/disable congestion alarm" @@ -102,14 +101,14 @@ emqx_schema { conn_congestion_min_alarm_sustain_duration { desc { - en: "Minimal time before clearing the alarm.\n\n" - "The alarm is cleared only when there's no pending data in\n" - "the queue, and at least `min_alarm_sustain_duration`\n" - "milliseconds passed since the last time we considered the connection \"congested\".\n\n" + en: "Minimal time before clearing the alarm.
" + "The alarm is cleared only when there's no pending data in
" + "the queue, and at least min_alarm_sustain_duration" + "milliseconds passed since the last time we considered the connection 'congested'.
" "This is to avoid clearing and raising the alarm again too often." - zh: "清除警报前的最短时间。\n\n" - "只有当队列中没有挂起的数据,并且连接至少被堵塞了 \"min_alarm_sustain_duration\" 毫秒时,\n" - "报警才会被清除。这是为了避免太频繁地清除和再次发出警报." + zh: "清除警报前的最短时间。
" + "只有当队列中没有挂起的数据,并且连接至少被堵塞了 min_alarm_sustain_duration 毫秒时,
" + "报警才会被清除。这是为了避免太频繁地清除和再次发出警报。" } label { en: "Sustain duration" @@ -120,10 +119,10 @@ emqx_schema { force_gc_enable { desc { en: "Enable forced garbage collection." - zh: "启用强制垃圾回收" + zh: "启用强制垃圾回收。" } label { - en: "Enable forced garbage collection." + en: "Enable forced garbage collection" zh: "启用强制垃圾回收" } } @@ -131,7 +130,7 @@ emqx_schema { force_gc_count { desc { en: "GC the process after this many received messages." - zh: "在进程收到多少消息之后,对此进程执行垃圾回收" + zh: "在进程收到多少消息之后,对此进程执行垃圾回收。" } label { en: "Process GC messages num" @@ -142,7 +141,7 @@ emqx_schema { force_gc_bytes { desc { en: "GC the process after specified number of bytes have passed through." - zh: "在进程处理过多少个字节之后,对此进程执行垃圾回收" + zh: "在进程处理过多少个字节之后,对此进程执行垃圾回收。" } label { en: "Process GC bytes" @@ -190,7 +189,7 @@ emqx_schema { sysmon_vm_long_gc { desc { en: "Enable Long GC monitoring." - zh: "启用长垃圾回收监控" + zh: "启用长垃圾回收监控。" } label { en: "Enable Long GC monitoring." @@ -201,7 +200,7 @@ emqx_schema { sysmon_vm_long_schedule { desc { en: "Enable Long Schedule monitoring." - zh: "启用长调度监控" + zh: "启用长调度监控。" } label { en: "Enable Long Schedule monitoring." @@ -212,7 +211,7 @@ emqx_schema { sysmon_vm_large_heap { desc { en: "Enable Large Heap monitoring." - zh: "启用大 heap 监控" + zh: "启用大 heap 监控。" } label { en: "Enable Large Heap monitoring." @@ -223,7 +222,7 @@ emqx_schema { sysmon_vm_busy_dist_port { desc { en: "Enable Busy Distribution Port monitoring." - zh: "启用分布式端口过忙监控" + zh: "启用分布式端口过忙监控。" } label { en: "Enable Busy Distribution Port monitoring." @@ -234,7 +233,7 @@ emqx_schema { sysmon_vm_busy_port { desc { en: "Enable Busy Port monitoring." - zh: "启用端口过忙监控" + zh: "启用端口过忙监控。" } label { en: "Enable Busy Port monitoring." @@ -316,7 +315,7 @@ emqx_schema { sysmon_top_num_items { desc { en: "The number of top processes per monitoring group" - zh: "每个监视组的顶级进程数" + zh: "每个监视组的顶级进程数。" } label { en: "Top num items" @@ -327,7 +326,7 @@ emqx_schema { sysmon_top_sample_interval { desc { en: "Specifies how often process top should be collected" - zh: "指定应收集进程顶部的频率" + zh: "指定应收集进程顶部的频率。" } label { en: "Top sample interval" @@ -339,7 +338,7 @@ emqx_schema { desc { en: "Stop collecting data when the number of processes\n" "in the VM exceeds this value" - zh: "当VM中的进程数超过此值时,停止收集数据" + zh: "当 VM 中的进程数超过此值时,停止收集数据。" } label { en: "Max procs" @@ -350,7 +349,7 @@ emqx_schema { sysmon_top_db_hostname { desc { en: "Hostname of the PostgreSQL database that collects the data points" - zh: "收集数据点的 PostgreSQL 数据库的主机名" + zh: "收集数据点的 PostgreSQL 数据库的主机名。" } label { en: "DB Hostname" @@ -360,8 +359,8 @@ emqx_schema { sysmon_top_db_port { desc { - en: "Port of the PostgreSQL database that collects the data points" - zh: "收集数据点的 PostgreSQL 数据库的端口" + en: "Port of the PostgreSQL database that collects the data points." + zh: "收集数据点的 PostgreSQL 数据库的端口。" } label { en: "DB Port" @@ -454,7 +453,7 @@ emqx_schema { flapping_detect_enable { desc { en: "Enable flapping connection detection feature." - zh: "启用抖动检测功能" + zh: "启用抖动检测功能。" } label: { en: "Enable flapping detection" @@ -465,7 +464,7 @@ emqx_schema { flapping_detect_max_count { desc { en: "The maximum number of disconnects allowed for a MQTT Client in `window_time`" - zh: "MQTT 客户端在\"窗口\"时间内允许的最大断开次数" + zh: "MQTT 客户端在“窗口”时间内允许的最大断开次数。" } label: { en: "Max count" @@ -487,11 +486,11 @@ emqx_schema { flapping_detect_ban_time { desc { en: "How long the flapping clientid will be banned." - zh: "抖动的客户端将会被禁止登陆多长时间" + zh: "抖动的客户端将会被禁止登录多长时间。" } label: { en: "Ban time" - zh: "禁止登陆时长" + zh: "禁止登录时长" } } @@ -605,7 +604,7 @@ emqx_schema { persistent_session_builtin_session_table { desc { en: "Performance tuning options for built-in session table." - zh: "用于内建会话表的性能调优参数" + zh: "用于内建会话表的性能调优参数。" } label: { en: "Persistent session" @@ -627,7 +626,7 @@ emqx_schema { persistent_session_builtin_messages_table { desc { en: "Performance tuning options for built-in messages table." - zh: "用于内建消息表的性能调优参数" + zh: "用于内建消息表的性能调优参数。" } label: { en: "Persistent messages" @@ -638,7 +637,7 @@ emqx_schema { stats_enable { desc { en: "Enable/disable statistic data collection." - zh: "启用/禁用统计数据收集功能" + zh: "启用/禁用统计数据收集功能。" } label: { en: "Enable/disable statistic data collection." @@ -780,8 +779,8 @@ mqtt 下所有的配置作为全局的默认值存在,它可以被 zone< mqtt_ignore_loop_deliver { desc { - en: """Ignore loop delivery of messages for MQTT v3.1.1/v3.1.0, similar to No Local subscription option in MQTT 5.0""" - zh: """是否为 MQTT v3.1.1/v3.1.0 客户端忽略投递自己发布的消息,类似于 MQTT 5.0 中的 No Local 订阅选项""" + en: """Ignore loop delivery of messages for MQTT v3.1.1/v3.1.0, similar to No Local subscription option in MQTT 5.0.""" + zh: """是否为 MQTT v3.1.1/v3.1.0 客户端忽略投递自己发布的消息,类似于 MQTT 5.0 中的 No Local 订阅选项。""" } label: { en: """Ignore Loop Deliver""" @@ -1000,7 +999,7 @@ Supported configurations are the following: - pem: Convert DER certificate content to PEM format as Username - md5: Take the MD5 value of the content of the DER or PEM certificate as Username """ - zh: """使用对端证书中的 CN, DN 字段或整个证书内容来作为用户名。仅适用于 TLS 连接。 + zh: """使用对端证书中的 CN、DN 字段或整个证书内容来作为用户名。仅适用于 TLS 连接。 目前支持配置为以下内容: - cn: 取证书的 CN 字段作为 Username - dn: 取证书的 DN 字段作为 Username @@ -1025,7 +1024,7 @@ Supported configurations are the following: - pem: Convert DER certificate content to PEM format as Client ID - md5: Take the MD5 value of the content of the DER or PEM certificate as Client ID """ - zh: """使用对端证书中的 CN, DN 字段或整个证书内容来作为客户端 ID。仅适用于 TLS 连接。 + zh: """使用对端证书中的 CN、DN 字段或整个证书内容来作为客户端 ID。仅适用于 TLS 连接。 目前支持配置为以下内容: - cn: 取证书的 CN 字段作为 Client ID - dn: 取证书的 DN 字段作为 Client ID @@ -1064,10 +1063,10 @@ Supported configurations are the following: """ zh: """Session 在集群中的锁策略。 - - `loca`: 仅锁本节点的 Session - - `one`: 任选一个其它节点加锁 - - `quorum`: 选择集群中半数以上的节点加锁 - - `all`: 选择所有节点加锁 + - `loca`:仅锁本节点的 Session; + - `one`:任选一个其它节点加锁; + - `quorum`:选择集群中半数以上的节点加锁; + - `all`:选择所有节点加锁。 """ } } @@ -1082,10 +1081,10 @@ Supported configurations are the following: """ zh: """共享订阅消息派发策略。 - - `random`: 随机挑选一个共享订阅者派发 - - `round_robin`: 使用 round-robin 策略派发 - - `sticky`: 总是使用上次选中的订阅者派发,直到它断开连接 - - `hash`: 使用发送者的 Client ID 进行 Hash 来选择订阅者 + - `random`:随机挑选一个共享订阅者派发; + - `round_robin`:使用 round-robin 策略派发; + - `sticky`:总是使用上次选中的订阅者派发,直到它断开连接; + - `hash`:使用发送者的 Client ID 进行 Hash 来选择订阅者。 """ } } @@ -1136,13 +1135,12 @@ until the subscriber disconnects. subscriber was not found, send to a random subscriber cluster-wide """ cn: """共享订阅的分发策略名称。 -- `random`: 随机选择一个组内成员; -- `round_robin`: 循环选择下一个成员; -- `round_robin_per_group`: 在共享组内循环选择下一个成员; -- `sticky`: 使用上一次选中的成员; -- `hash`: 根据 ClientID 哈希映射到一个成员; -- `local`: 随机分发到节点本地成成员,如果本地成员不存在,则随机分发 -到任意一个成员。 +- `random`:随机选择一个组内成员; +- `round_robin`:循环选择下一个成员; +- `round_robin_per_group`:在共享组内循环选择下一个成员; +- `sticky`:使用上一次选中的成员; +- `hash`:根据 ClientID 哈希映射到一个成员; +- `local`:随机分发到节点本地成成员,如果本地成员不存在,则随机分发到任意一个成员。 """ } @@ -1162,9 +1160,9 @@ NOTE: when changing from/to `global` lock, it requires all nodes in the cluster 建议仅当通配符主题较多时才更改此参数。 注:当从/更改为 `global` 锁时,它要求集群中的所有节点在更改之前停止。 - - `key`: 为 Mnesia 事务涉及到的每个 key 上锁,建议单节点时使用。 - - `tab`: 为 Mnesia 事务涉及到的表上锁,建议在集群中使用。 - - `global`: 所以更新操作都被全局的锁保护,仅建议在超大规模集群中使用。 + - `key`:为 Mnesia 事务涉及到的每个 key 上锁,建议单节点时使用。 + - `tab`:为 Mnesia 事务涉及到的表上锁,建议在集群中使用。 + - `global`:所以更新操作都被全局的锁保护,仅建议在超大规模集群中使用。 """ } } @@ -1215,8 +1213,8 @@ NOTE: This is a cluster-wide configuration. It requires all nodes to be stopped sys_event_messages { desc { - en: """Client events messages""" - zh: """客户端事件消息""" + en: """Client events messages.""" + zh: """客户端事件消息。""" } } @@ -1271,12 +1269,8 @@ Find more details in 'authorization.sources' config. fields_authorization_deny_action { desc { - en: """ -The action when the authorization check rejects an operation. -""" - zh: """ -授权检查拒绝操作时的操作。 -""" + en: """The action when the authorization check rejects an operation.""" + zh: """授权检查拒绝操作时的操作。""" } label: { en: "Authorization deny action" @@ -1286,12 +1280,8 @@ The action when the authorization check rejects an operation. fields_cache_enable { desc { - en: """ -Enable or disable the authorization cache. -""" - zh: """ -启用或禁用授权缓存。 -""" + en: """Enable or disable the authorization cache.""" + zh: """启用或禁用授权缓存。""" } label: { en: "Enable or disable the authorization cache." @@ -1301,12 +1291,8 @@ Enable or disable the authorization cache. fields_cache_max_size { desc { - en: """ -Maximum number of cached items. -""" - zh: """ -缓存项的最大数量。 -""" + en: """Maximum number of cached items.""" + zh: """缓存项的最大数量。""" } label: { en: "Maximum number of cached items." @@ -1316,12 +1302,8 @@ Maximum number of cached items. fields_cache_ttl { desc { - en: """ -Time to live for the cached data. -""" - zh: """ -缓存数据的生存时间。 -""" + en: """Time to live for the cached data. """ + zh: """缓存数据的生存时间。""" } label: { en: "Time to live for the cached data." @@ -1331,9 +1313,7 @@ Time to live for the cached data. fields_deflate_opts_level { desc { - en: """ -Compression level. -""" + en: """Compression level. """ zh: """压缩级别""" } label: { @@ -1361,12 +1341,8 @@ Lower values decrease memory usage per connection. fields_deflate_opts_strategy { desc { - en: """ -Specifies the compression strategy. -""" - zh: """ -指定压缩策略。 -""" + en: """Specifies the compression strategy.""" + zh: """指定压缩策略。""" } label: { en: "compression strategy" @@ -1376,9 +1352,7 @@ Specifies the compression strategy. fields_deflate_opts_server_context_takeover { desc { - en: """ -Takeover means the compression state is retained between server messages. -""" + en: """Takeover means the compression state is retained between server messages. """ zh: """接管意味着在服务器消息之间保留压缩状态。""" } label: { @@ -1389,12 +1363,8 @@ Takeover means the compression state is retained between server messages. fields_deflate_opts_client_context_takeover { desc { - en: """ -Takeover means the compression state is retained between client messages. -""" - zh: """ -接管意味着在客户端消息之间保留压缩状态。 -""" + en: """Takeover means the compression state is retained between client messages. """ + zh: """接管意味着在客户端消息之间保留压缩状态。""" } label: { en: "Client context takeover" @@ -1404,12 +1374,8 @@ Takeover means the compression state is retained between client messages. fields_deflate_opts_server_max_window_bits { desc { - en: """ -Specifies the size of the compression context for the server. -""" - zh: """ -指定服务器压缩上下文的大小。 -""" + en: """Specifies the size of the compression context for the server.""" + zh: """指定服务器压缩上下文的大小。""" } label: { en: "Server compression max window size" @@ -1419,12 +1385,8 @@ Specifies the size of the compression context for the server. fields_deflate_opts_client_max_window_bits { desc { - en: """ -Specifies the size of the compression context for the client. -""" - zh: """ -指定客户端压缩上下文的大小。 -""" + en: """Specifies the size of the compression context for the client.""" + zh: """指定客户端压缩上下文的大小。""" } label: { en: "Client compression max window size" @@ -1434,10 +1396,8 @@ Specifies the size of the compression context for the client. client_ssl_opts_schema_enable { desc { - en: """ -Enable TLS. -""" - zh: """启用 TLS""" + en: """Enable TLS. """ + zh: """启用 TLS。""" } label: { en: "Enable TLS." @@ -1495,12 +1455,8 @@ PEM格式证书链文件
common_ssl_opts_schema_keyfile { desc { - en: """ -PEM format private key file. -""" - zh: """ -PEM格式的私钥文件。 -""" + en: """PEM format private key file. """ + zh: """PEM格式的私钥文件。""" } label: { en: "Keyfile" @@ -1510,12 +1466,8 @@ PEM格式的私钥文件。 common_ssl_opts_schema_verify { desc { - en: """ -Enable or disable peer verification. -""" - zh: """ -启用或禁用对等验证。 -""" + en: """Enable or disable peer verification. """ + zh: """启用或禁用对等验证。""" } label: { en: "Verify peer" @@ -1525,12 +1477,8 @@ Enable or disable peer verification. common_ssl_opts_schema_reuse_sessions { desc { - en: """ -Enable TLS session reuse. -""" - zh: """ -启用 TLS 会话重用。 -""" + en: """Enable TLS session reuse. """ + zh: """启用 TLS 会话重用。""" } label: { en: "TLS session reuse" @@ -1541,7 +1489,8 @@ Enable TLS session reuse. common_ssl_opts_schema_depth { desc { en: """ -Maximum number of non-self-issued intermediate certificates that can follow the peer certificate in a valid certification path. So, if depth is 0 the PEER must be signed by the trusted ROOT-CA directly; if 1 the path can be PEER, CA, ROOT-CA; if 2 the path can be PEER, CA, CA, ROOT-CA, and so on. The default value is 10. +Maximum number of non-self-issued intermediate certificates that can follow the peer certificate in a valid certification path. +So, if depth is 0 the PEER must be signed by the trusted ROOT-CA directly; if 1 the path can be PEER, CA, ROOT-CA; if 2 the path can be PEER, CA, CA, ROOT-CA, and so on. The default value is 10. """ zh: """ 在有效的证书路径中,可以跟随对等证书的非自颁发中间证书的最大数量。因此,如果深度为0,则对等方必须由受信任的根CA直接签名;如果1,路径可以是PEER、CA、ROOT-CA;如果是2,则路径可以是PEER、CA、CA、ROOT-CA等等。默认值为10。 @@ -1724,12 +1673,8 @@ RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"

common_ssl_opts_schema_user_lookup_fun { desc { - en: """ -EMQX-internal callback that is used to lookup pre-shared key (PSK) identity. -""" - zh: """ -用于查找预共享密钥(PSK)标识的 EMQX 内部回调。 -""" + en: """EMQX-internal callback that is used to lookup pre-shared key (PSK) identity. """ + zh: """用于查找预共享密钥(PSK)标识的 EMQX 内部回调。""" } label: { en: "SSL PSK user lookup fun" @@ -1766,7 +1711,7 @@ are used.
NOTE: The dhfile option is not supported by TLS 1.3. """ zh: """ -如果协商使用Diffie-Hellman密钥交换的密码套件,则服务器将使用包含PEM编码的Diffie-Hellman参数的文件的路径。如果未指定,则使用默认参数
+如果协商使用Diffie-Hellman密钥交换的密码套件,则服务器将使用包含PEM编码的Diffie-Hellman参数的文件的路径。如果未指定,则使用默认参数。
注意:TLS 1.3不支持dhfile选项。 """ @@ -1875,10 +1820,8 @@ TLS/SSL握手建立后立即进行GC。 fields_listeners_tcp { desc { - en: """ -TCP listeners -""" - zh: """TCP 监听器""" + en: """TCP listeners.""" + zh: """TCP 监听器。""" } label: { en: "TCP listeners" @@ -1888,10 +1831,8 @@ TCP listeners fields_listeners_ssl { desc { - en: """ -SSL listeners -""" - zh: """SSL 监听器""" + en: """SSL listeners.""" + zh: """SSL 监听器。""" } label: { en: "SSL listeners" @@ -1901,10 +1842,8 @@ SSL listeners fields_listeners_ws { desc { - en: """ -HTTP websocket listeners -""" - zh: """HTTP websocket 监听器""" + en: """HTTP websocket listeners.""" + zh: """HTTP websocket 监听器。""" } label: { en: "HTTP websocket listeners" @@ -1914,10 +1853,8 @@ HTTP websocket listeners fields_listeners_wss { desc { - en: """ -HTTPS websocket listeners -""" - zh: """HTTPS websocket 监听器""" + en: """HTTPS websocket listeners.""" + zh: """HTTPS websocket 监听器。""" } label: { en: "HTTPS websocket listeners" @@ -1927,10 +1864,8 @@ HTTPS websocket listeners fields_listeners_quic { desc { - en: """ -QUIC listeners -""" - zh: """QUIC 监听器""" + en: """QUIC listeners.""" + zh: """QUIC 监听器。""" } label: { en: "QUIC listeners" @@ -1940,10 +1875,8 @@ QUIC listeners fields_listener_enabled { desc { - en: """ -Enable listener. -""" - zh: """启停监听器""" + en: """Enable listener. """ + zh: """启停监听器。""" } label: { en: "Enable listener" @@ -1953,10 +1886,8 @@ Enable listener. fields_mqtt_quic_listener_certfile { desc { - en: """ -Path to the certificate file. -""" - zh: """证书文件""" + en: """Path to the certificate file.""" + zh: """证书文件。""" } label: { en: "Certificate file" @@ -1966,10 +1897,8 @@ Path to the certificate file. fields_mqtt_quic_listener_keyfile { desc { - en: """ -Path to the secret key file. -""" - zh: """私钥文件""" + en: """Path to the secret key file. """ + zh: """私钥文件。""" } label: { en: "Key file" @@ -1979,31 +1908,23 @@ Path to the secret key file. fields_mqtt_quic_listener_idle_timeout { desc { - en: """ -How long a connection can go idle before it is gracefully shut down. 0 to disable -""" - zh: """ -一个连接在被关闭之前可以空闲多长时间。0表示禁用 -""" + en: """How long a connection can go idle before it is gracefully shut down. 0 to disable""" + zh: """一个连接在被关闭之前可以空闲多长时间。0表示禁用。""" } label: { en: "Idle Timeout" - zh: "发呆超时时间" + zh: "空闲超时时间" } } fields_mqtt_quic_listener_handshake_idle_timeout { desc { - en: """ -How long a handshake can idle before it is discarded. -""" - zh: """ -一个握手在被丢弃之前可以空闲多长时间。 -""" + en: """How long a handshake can idle before it is discarded. """ + zh: """一个握手在被丢弃之前可以空闲多长时间。""" } label: { en: "Handshake Idle Timeout" - zh: "握手发呆超时时间" + zh: "握手空闲超时时间" } } @@ -2013,7 +1934,7 @@ fields_mqtt_quic_listener_keep_alive_interval { How often to send PING frames to keep a connection alive. 0 means disabled. """ zh: """ -发送 PING 帧的频率,以保活连接. 设为0,禁用 +发送 PING 帧的频率,以保活连接. 设为 0 表示禁用。 """ } label: { @@ -2039,9 +1960,7 @@ IP address and port for the listening socket. base_listener_acceptors { desc { - en: """ -The size of the listener's receiving pool. -""" + en: """The size of the listener's receiving pool.""" zh: """监听器接收池的大小。""" } label: { @@ -2052,12 +1971,8 @@ The size of the listener's receiving pool. base_listener_max_connections { desc { - en: """ -The maximum number of concurrent connections allowed by the listener. -""" - zh: """ -监听器允许的最大并发连接数。 -""" + en: """The maximum number of concurrent connections allowed by the listener. """ + zh: """监听器允许的最大并发连接数。""" } label: { en: "Max connections" @@ -2218,7 +2133,7 @@ If there is no decision after a full chain exhaustion, the login is rejected. 该配置可以被配置为:
    -
  • []: 默认值,允许所以登录请求 +
  • []: 默认值,允许所有的登录请求
  • 配置为单认证器,例如 {enable:true,backend:\"built_in_database\",mechanism=\"password_based\"}
  • 配置为认证器数组
@@ -2242,7 +2157,7 @@ When authenticating a login (username, client ID, etc.) the authenticators are c 监听器认证重载。 认证配置可以是单个认证器实例,也可以是一个认证器数组组成的认证链。 -执行登录验证时(用户名、客户端 ID 等),将按配置的顺序执行
+执行登录验证时(用户名、客户端 ID 等),将按配置的顺序执行。 """ } label: { @@ -2253,12 +2168,8 @@ When authenticating a login (username, client ID, etc.) the authenticators are c fields_rate_limit_max_conn_rate { desc { - en: """ -Maximum connections per second. -""" - zh: """ -每秒最大连接数。 -""" + en: """Maximum connections per second.""" + zh: """每秒最大连接数。""" } label: { en: "Max connection rate" @@ -2268,12 +2179,8 @@ Maximum connections per second. fields_rate_limit_conn_messages_in { desc { - en: """ -Message limit for the external MQTT connections. -""" - zh: """ -外部 MQTT 连接的消息限制。 -""" + en: """Message limit for the external MQTT connections.""" + zh: """外部 MQTT 连接的消息限制。""" } label: { en: "connecting messages in" @@ -2314,13 +2221,13 @@ Indication extension from being sent and disables the hostname verification check. """ zh: """ -指定要在 TLS 服务器名称指示扩展中使用的主机名
+指定要在 TLS 服务器名称指示扩展中使用的主机名。
例如,当连接到 "server.example.net" 时,接受连接并执行 TLS 握手的真正服务器可能与 TLS 客户端最初连接到的主机不同, 例如,当连接到 IP 地址时,或者当主机具有多个可解析的 DNS 记录时
如果未指定,它将默认为使用的主机名字符串 建立连接,除非使用 IP 地址
-然后,主机名也用于对等机的主机名验证 -证书
特殊值'disable'阻止发送服务器名称指示扩展,并禁用主机名验证检查。 +然后,主机名也用于对等机的主机名验证证书
+特殊值 disable 阻止发送服务器名称指示扩展,并禁用主机名验证检查。 """ } label: { @@ -2364,12 +2271,8 @@ TCP backlog 定义了挂起连接队列可以增长到的最大长度。 fields_tcp_opts_send_timeout { desc { - en: """ -The TCP send timeout for the connections. -""" - zh: """ -连接的TCP发送超时。 -""" + en: """The TCP send timeout for the connections. """ + zh: """连接的 TCP 发送超时。""" } label: { en: "TCP send timeout" @@ -2398,7 +2301,7 @@ fields_tcp_opts_recbuf { The TCP receive buffer (OS kernel) for the connections. """ zh: """ -连接的 TCP 接收缓冲区(OS内核)。 +连接的 TCP 接收缓冲区(OS 内核)。 """ } label: { @@ -2413,7 +2316,7 @@ fields_tcp_opts_sndbuf { The TCP send buffer (OS kernel) for the connections. """ zh: """ -连接的 TCP 发送缓冲区(OS内核)。 +连接的 TCP 发送缓冲区(OS 内核)。 """ } label: { @@ -2494,7 +2397,7 @@ Determine the format of the payload format in the trace file.
""" zh: """ -确定跟踪文件中有效负载格式的格式
+确定跟踪文件中有效负载格式的格式。
`text`:基于文本的协议或纯文本协议。 建议在有效负载为JSON编码时使用
`hex`:二进制十六进制编码。当有效负载是自定义二进制协议时,建议使用此选项
@@ -2568,7 +2471,7 @@ message within this interval. } label: { en: "WS idle timeout" - zh: "WS 发呆时间" + zh: "WS 空闲时间" } } diff --git a/apps/emqx_authn/i18n/emqx_authn_jwt_i18n.conf b/apps/emqx_authn/i18n/emqx_authn_jwt_i18n.conf index 80d389de4..a420dd7d9 100644 --- a/apps/emqx_authn/i18n/emqx_authn_jwt_i18n.conf +++ b/apps/emqx_authn/i18n/emqx_authn_jwt_i18n.conf @@ -57,7 +57,7 @@ emqx_authn_jwt { endpoint { desc { en: """JWKS endpoint, it's a read-only endpoint that returns the server's public key set in the JWKS format.""" - zh: """JWKS 端点, 它是一个以 JWKS 格式返回服务端的公钥集的只读端点。""" + zh: """JWKS 端点, 它是一个以 JWKS 格式返回服务端的公钥集的只读端点。""" } label { en: """JWKS Endpoint""" diff --git a/apps/emqx_authz/i18n/emqx_authz_api_cache_i18n.conf b/apps/emqx_authz/i18n/emqx_authz_api_cache_i18n.conf index 31ad90fa1..ea6960555 100644 --- a/apps/emqx_authz/i18n/emqx_authz_api_cache_i18n.conf +++ b/apps/emqx_authz/i18n/emqx_authz_api_cache_i18n.conf @@ -2,7 +2,7 @@ emqx_authz_api_cache { authorization_cache_delete { desc { en: """Clean all authorization cache in the cluster.""" - zh: """清除集群中所有鉴权数据缓存""" + zh: """清除集群中所有授权数据缓存""" } } } diff --git a/apps/emqx_authz/i18n/emqx_authz_api_schema_i18n.conf b/apps/emqx_authz/i18n/emqx_authz_api_schema_i18n.conf index 6d2c66b79..681cad7a2 100644 --- a/apps/emqx_authz/i18n/emqx_authz_api_schema_i18n.conf +++ b/apps/emqx_authz/i18n/emqx_authz_api_schema_i18n.conf @@ -26,7 +26,7 @@ emqx_authz_api_schema { rules { desc { en: """Authorization static file rules.""" - zh: """静态鉴权文件规则""" + zh: """静态授权文件规则""" } label { en: """rules""" @@ -111,7 +111,7 @@ emqx_authz_api_schema { collection { desc { en: """`MongoDB` collection containing the authorization data.""" - zh: """`MongoDB` 鉴权数据集""" + zh: """`MongoDB` 授权数据集""" } label { en: """collection""" diff --git a/apps/emqx_authz/i18n/emqx_authz_api_settings_i18n.conf b/apps/emqx_authz/i18n/emqx_authz_api_settings_i18n.conf index c739bedbb..b44580b34 100644 --- a/apps/emqx_authz/i18n/emqx_authz_api_settings_i18n.conf +++ b/apps/emqx_authz/i18n/emqx_authz_api_settings_i18n.conf @@ -2,14 +2,14 @@ emqx_authz_api_settings { authorization_settings_get { desc { en: """Get authorization settings""" - zh: """获取鉴权配置""" + zh: """获取授权配置""" } } authorization_settings_put { desc { en: """Update authorization settings""" - zh: """更新鉴权配置""" + zh: """更新授权配置""" } } } diff --git a/apps/emqx_authz/i18n/emqx_authz_api_sources_i18n.conf b/apps/emqx_authz/i18n/emqx_authz_api_sources_i18n.conf index 6c3f59d93..c5f0eaad4 100644 --- a/apps/emqx_authz/i18n/emqx_authz_api_sources_i18n.conf +++ b/apps/emqx_authz/i18n/emqx_authz_api_sources_i18n.conf @@ -2,56 +2,56 @@ emqx_authz_api_sources { authorization_sources_get { desc { en: """List all authorization sources""" - zh: """列出所有鉴权数据源""" + zh: """列出所有授权数据源""" } } authorization_sources_post { desc { en: """Add a new source""" - zh: """添加鉴权数据源""" + zh: """添加授权数据源""" } } authorization_sources_type_get { desc { en: """Get a authorization source""" - zh: """获取指定类型的鉴权数据源""" + zh: """获取指定类型的授权数据源""" } } authorization_sources_type_put { desc { en: """Update source""" - zh: """更新指定类型的鉴权数据源""" + zh: """更新指定类型的授权数据源""" } } authorization_sources_type_delete { desc { en: """Delete source""" - zh: """删除指定类型的鉴权数据源""" + zh: """删除指定类型的授权数据源""" } } authorization_sources_type_status_get { desc { en: """Get a authorization source""" - zh: """获取指定鉴权数据源的状态""" + zh: """获取指定授权数据源的状态""" } } authorization_sources_type_move_post { desc { en: """Change the exection order of sources""" - zh: """更新鉴权数据源的优先执行顺序""" + zh: """更新授权数据源的优先执行顺序""" } } sources { desc { en: """Authorization source""" - zh: """鉴权数据源列表""" + zh: """授权数据源列表""" } label { en: """sources""" @@ -62,7 +62,7 @@ emqx_authz_api_sources { sources { desc { en: """Authorization sources""" - zh: """鉴权数据源列表""" + zh: """授权数据源列表""" } label { en: """sources""" @@ -84,7 +84,7 @@ emqx_authz_api_sources { source { desc { en: """Authorization source""" - zh: """鉴权数据源""" + zh: """授权数据源""" } label { en: """source""" diff --git a/apps/emqx_authz/i18n/emqx_authz_schema_i18n.conf b/apps/emqx_authz/i18n/emqx_authz_schema_i18n.conf index e83169978..04118353e 100644 --- a/apps/emqx_authz/i18n/emqx_authz_schema_i18n.conf +++ b/apps/emqx_authz/i18n/emqx_authz_schema_i18n.conf @@ -21,22 +21,22 @@ The source elements are identified by their 'type'. It is NOT allowed to configure two or more sources of the same type. """ zh: """ -鉴权数据源.
-鉴权(ACL)数据源的列表. -它被设计为一个数组,而不是一个散列映射, -所以可以作为链式访问控制.
+授权数据源。
+授权(ACL)数据源的列表。 +它被设计为一个数组,而不是一个散列映射, +所以可以作为链式访问控制。
-当授权一个 'publish' 或 'subscribe' 行为时, +当授权一个 'publish' 或 'subscribe' 行为时, 该配置列表中的所有数据源将按顺序进行检查。 -如果在某个客户端未找到时(使用 ClientID 或 Username), -将会移动到下一个数据源. 直至得到 'allow' 或 'deny' 的结果.
+如果在某个客户端未找到时(使用 ClientID 或 Username)。 +将会移动到下一个数据源。直至得到 'allow' 或 'deny' 的结果。
-如果在任何数据源中都未找到对应的客户端信息, -配置的默认行为 ('authorization.no_match') 将生效.
+如果在任何数据源中都未找到对应的客户端信息。 +配置的默认行为 ('authorization.no_match') 将生效。
-注意: -数据源使用 'type' 进行标识. -使用同一类型的数据源多于一次不被允许. +注意: +数据源使用 'type' 进行标识。 +使用同一类型的数据源多于一次不被允许。 """ } label { @@ -83,7 +83,7 @@ It is NOT allowed to configure two or more sources of the same type. file { desc { en: """Authorization using a static file.""" - zh: """使用静态文件鉴权""" + zh: """使用静态文件授权""" } label { en: """file""" @@ -109,7 +109,7 @@ and the old file will not be used anymore. 那么可以将该文件置于任何 EMQX 可以访问到的位置。 如果从 EMQX Dashboard 或 HTTP API 创建或修改了规则集, -那么EMQX将会生成一个新的文件并将它存放在 `data_dir` 下的 `authz` 子目录中, +那么EMQX将会生成一个新的文件并将它存放在 `data_dir` 下的 `authz` 子目录中, 并从此弃用旧的文件。""" } label { @@ -123,7 +123,7 @@ and the old file will not be used anymore. http_get { desc { en: """Authorization using an external HTTP server (via GET requests).""" - zh: """使用外部 HTTP 服务器鉴权(GET 请求)。""" + zh: """使用外部 HTTP 服务器授权(GET 请求)。""" } label { en: """http_get""" @@ -134,7 +134,7 @@ and the old file will not be used anymore. http_post { desc { en: """Authorization using an external HTTP server (via POST requests).""" - zh: """使用外部 HTTP 服务器鉴权(POST 请求)。""" + zh: """使用外部 HTTP 服务器授权(POST 请求)。""" } label { en: """http_post""" @@ -156,7 +156,7 @@ and the old file will not be used anymore. url { desc { en: """URL of the auth server.""" - zh: """鉴权 HTTP 服务器地址。""" + zh: """授权 HTTP 服务器地址。""" } label { en: """URL""" @@ -213,7 +213,7 @@ and the old file will not be used anymore. mnesia { desc { en: """Authorization using a built-in database (mnesia).""" - zh: """使用内部数据库鉴权 (mnesia).""" + zh: """使用内部数据库授权(mnesia)。""" } label { en: """mnesia""" @@ -226,7 +226,7 @@ and the old file will not be used anymore. mongo_single { desc { en: """Authorization using a single MongoDB instance.""" - zh: """使用 MongoDB 鉴权(单实例)""" + zh: """使用 MongoDB 授权(单实例)。""" } label { en: """mongo_single""" @@ -237,7 +237,7 @@ and the old file will not be used anymore. mongo_rs { desc { en: """Authorization using a MongoDB replica set.""" - zh: """使用 MongoDB 鉴权(副本集模式)""" + zh: """使用 MongoDB 授权(副本集模式)""" } label { en: """mongo_rs""" @@ -248,7 +248,7 @@ and the old file will not be used anymore. mongo_sharded { desc { en: """Authorization using a sharded MongoDB cluster.""" - zh: """使用 MongoDB 鉴权(分片集群模式)""" + zh: """使用 MongoDB 授权(分片集群模式)。""" } label { en: """mongo_sharded""" @@ -259,7 +259,7 @@ and the old file will not be used anymore. collection { desc { en: """`MongoDB` collection containing the authorization data.""" - zh: """`MongoDB` 鉴权数据集""" + zh: """`MongoDB` 授权数据集。""" } label { en: """collection""" @@ -278,8 +278,8 @@ Filter supports the following placeholders: zh: """ 在查询中定义过滤条件的条件表达式。 过滤器支持如下占位符: -- ${username}: 将在运行时被替换为客户端连接时使用的用户名 -- ${clientid}: 将在运行时被替换为客户端连接时使用的客户端标识符 +- ${username}:将在运行时被替换为客户端连接时使用的用户名 +- ${clientid}:将在运行时被替换为客户端连接时使用的客户端标识符 """ } label { @@ -293,7 +293,7 @@ Filter supports the following placeholders: mysql { desc { en: """Authorization using a MySQL database.""" - zh: """使用 MySOL 数据库鉴权""" + zh: """使用 MySOL 数据库授权""" } label { en: """mysql""" @@ -306,7 +306,7 @@ Filter supports the following placeholders: postgresql { desc { en: """Authorization using a PostgreSQL database.""" - zh: """使用 PostgreSQL 数据库鉴权""" + zh: """使用 PostgreSQL 数据库授权""" } label { en: """postgresql""" @@ -319,7 +319,7 @@ Filter supports the following placeholders: redis_single { desc { en: """Authorization using a single Redis instance.""" - zh: """使用 Redis 鉴权(单实例)""" + zh: """使用 Redis 授权(单实例)。""" } label { en: """redis_single""" @@ -330,7 +330,7 @@ Filter supports the following placeholders: redis_sentinel { desc { en: """Authorization using a Redis Sentinel.""" - zh: """使用 Redis 鉴权(哨兵模式)""" + zh: """使用 Redis 授权(哨兵模式)。""" } label { en: """redis_sentinel""" @@ -341,7 +341,7 @@ Filter supports the following placeholders: redis_cluster { desc { en: """Authorization using a Redis cluster.""" - zh: """使用 Redis 鉴权(集群模式)""" + zh: """使用 Redis 授权(集群模式)。""" } label { en: """redis_cluster""" @@ -365,7 +365,7 @@ Filter supports the following placeholders: query { desc { en: """Database query used to retrieve authorization data.""" - zh: """访问控制数据查询语句/查询命令""" + zh: """访问控制数据查询语句/查询命令。""" } label { en: """query""" @@ -510,44 +510,44 @@ Filter supports the following placeholders: metrics_total { desc { en: """The total number of times the authorization rule was triggered.""" - zh: """鉴权实例被触发的总次数。""" + zh: """授权实例被触发的总次数。""" } label: { en: """The Total Number of Times the Authorization Rule was Triggered""" - zh: """鉴权实例被触发的总次数""" + zh: """授权实例被触发的总次数""" } } nomatch { desc { en: """The number of times that no authorization rules were matched.""" - zh: """没有匹配到任何鉴权规则的次数。""" + zh: """没有匹配到任何授权规则的次数。""" } label: { en: """The Number of Times that no Authorization Rules were Matched""" - zh: """没有匹配到任何鉴权规则的次数""" + zh: """没有匹配到任何授权规则的次数""" } } allow { desc { en: """The number of times the authentication was successful.""" - zh: """鉴权成功的次数。""" + zh: """授权成功的次数。""" } label: { en: """The Number of Times the Authentication was Successful""" - zh: """鉴权成功次数""" + zh: """授权成功次数""" } } deny { desc { en: """The number of authentication failures.""" - zh: """鉴权失败的次数。""" + zh: """授权失败的次数。""" } label: { en: """The Number of Authentication Failures""" - zh: """鉴权失败次数""" + zh: """授权失败次数""" } } } diff --git a/apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_i18n.conf b/apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_i18n.conf index b93a186b1..57f744e8e 100644 --- a/apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_i18n.conf +++ b/apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_i18n.conf @@ -2,7 +2,7 @@ emqx_auto_subscribe_schema { auto_subscribe { desc { en: """After the device logs in successfully, the subscription is automatically completed for the device through the pre-defined subscription representation. Supports the use of placeholders.""" - zh: """设备登陆成功之后,通过预设的订阅表示符,为设备自动完成订阅。支持使用占位符。""" + zh: """设备登录成功之后,通过预设的订阅表示符,为设备自动完成订阅。支持使用占位符。""" } lable { en: """Auto Subscribe""" diff --git a/apps/emqx_conf/i18n/emqx_conf_schema.conf b/apps/emqx_conf/i18n/emqx_conf_schema.conf index 101823d5f..26f3ac9b5 100644 --- a/apps/emqx_conf/i18n/emqx_conf_schema.conf +++ b/apps/emqx_conf/i18n/emqx_conf_schema.conf @@ -357,7 +357,7 @@ Setting cluster.k8s.address_type to ip will make EMQX to discover IP addresses of peer nodes from Kubernetes API. """ zh: """当使用 k8s 方式集群时,address_type 用来从 Kubernetes 接口的应答里获取什么形式的 Host 列表。 -指定 cluster.k8s.address_typeip,则将从 Kubernetes 接口中获取集群中其他节点 +指定 cluster.k8s.address_typeip,则将从 Kubernetes 接口中获取集群中其他节点 的IP地址。 """ } @@ -1201,7 +1201,7 @@ Supervisor 报告的类型。默认为 error 类型。 desc { en: """Enable log rotation feature.""" zh: """启用日志轮换功能。启动后生成日志文件后缀会加上对应的索引数字,比如:log/emqx.log.1。 -系统会默认生成*.siz/*.idx用于记录日志位置,请不要手动修改这两个文件。 +系统会默认生成*.siz/*.idx用于记录日志位置,请不要手动修改这两个文件。 """ } label { diff --git a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf b/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf index c9f227383..99ae420f1 100644 --- a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf +++ b/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf @@ -9,15 +9,15 @@ In 'cluster_shareload' mode, the incoming load from the remote broker is shared using shared subscription.
Note that the 'clientid' is suffixed by the node name, this is to avoid clientid conflicts between different nodes. And we can only use shared subscription -topic filters for 'remote_topic' of ingress connections. +topic filters for remote_topic of ingress connections. """ zh: """ MQTT 桥的模式。
- cluster_shareload:在 emqx 集群的每个节点上创建一个 MQTT 连接。
在“cluster_shareload”模式下,来自远程代理的传入负载通过共享订阅的方式接收。
-请注意,“clientid”以节点名称为后缀,这是为了避免不同节点之间的clientid冲突。 -而且对于入口连接的“remote_topic”,我们只能使用共享订阅主题过滤器。 +请注意,clientid 以节点名称为后缀,这是为了避免不同节点之间的clientid冲突。 +而且对于入口连接的 remote_topic,我们只能使用共享订阅主题过滤器。 """ } label: { diff --git a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf index 32d8cfe48..a05fec5c4 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf @@ -146,7 +146,7 @@ This option specifies the QoS level for the CoAP Client when establishing a subs """ zh: """客户端订阅请求的默认 QoS 等级。 当 CoAP 客户端发起订阅请求时,如果未携带 `qos` 参数则会使用该默认值。默认值可设置为: - - qos0, qos1, qos2: 设置为固定的 QoS 等级 + - qos0、 qos1、qos2: 设置为固定的 QoS 等级 - coap: 依据订阅操作的 CoAP 报文类型来动态决定 * 当订阅请求为 `non-confirmable` 类型时,取值为 qos0 * 当订阅请求为 `confirmable` 类型时,取值为 qos1 @@ -165,7 +165,7 @@ This option specifies the QoS level for the CoAP Client when publishing a messag zh: """客户端发布请求的默认 QoS 等级。 当 CoAP 客户端发起发布请求时,如果未携带 `qos` 参数则会使用该默认值。默认值可设置为: - - qos0, qos1, qos2: 设置为固定的 QoS 等级 + - qos0、qos1、qos2: 设置为固定的 QoS 等级 - coap: 依据发布操作的 CoAP 报文类型来动态决定 * 当发布请求为 `non-confirmable` 类型时,取值为 qos0 * 当发布请求为 `confirmable` 类型时,取值为 qos1 @@ -175,29 +175,29 @@ This option specifies the QoS level for the CoAP Client when publishing a messag lwm2m { desc { - en: """The LwM2M Gateway configuration. This gateway only supports the v1.0.1 protocol""" - zh: """LwM2M 网关配置。仅支持 v1.0.1 协议""" + en: """The LwM2M Gateway configuration. This gateway only supports the v1.0.1 protocol.""" + zh: """LwM2M 网关配置。仅支持 v1.0.1 协议。""" } } lwm2m_xml_dir { desc { - en: """The Directory for LwM2M Resource definition""" - zh: """LwM2M Resource 定义的 XML 文件目录路径""" + en: """The Directory for LwM2M Resource definition.""" + zh: """LwM2M Resource 定义的 XML 文件目录路径。""" } } lwm2m_lifetime_min { desc { - en: """Minimum value of lifetime allowed to be set by the LwM2M client""" - zh: """允许 LwM2M 客户端允许设置的心跳最小值""" + en: """Minimum value of lifetime allowed to be set by the LwM2M client.""" + zh: """允许 LwM2M 客户端允许设置的心跳最小值。""" } } lwm2m_lifetime_max { desc { - en: """Maximum value of lifetime allowed to be set by the LwM2M client""" - zh: """允许 LwM2M 客户端允许设置的心跳最大值""" + en: """Maximum value of lifetime allowed to be set by the LwM2M client.""" + zh: """允许 LwM2M 客户端允许设置的心跳最大值。""" } } @@ -207,14 +207,14 @@ This option specifies the QoS level for the CoAP Client when publishing a messag For example, after receiving an update message from a client, any messages within this time window are sent directly to the LwM2M client, and all messages beyond this time window are temporarily stored in memory.""" zh: """在QMode模式下,LwM2M网关认为网络链接有效的时间窗口的值。 -例如,在收到客户端的更新信息后,在这个时间窗口内的任何信息都会直接发送到LwM2M客户端,而超过这个时间窗口的所有信息都会暂时储存在内存中""" +例如,在收到客户端的更新信息后,在这个时间窗口内的任何信息都会直接发送到LwM2M客户端,而超过这个时间窗口的所有信息都会暂时储存在内存中。""" } } lwm2m_auto_observe { desc { - en: """Automatically observe the object list of REGISTER packet""" - zh: """自动 Observe REGISTER 数据包的 Object 列表""" + en: """Automatically observe the object list of REGISTER packet.""" + zh: """自动 Observe REGISTER 数据包的 Object 列表。""" } } @@ -226,15 +226,15 @@ For example, after receiving an update message from a client, any messages withi """ zh: """发布UPDATE事件消息的策略。 - always: 只要收到 UPDATE 请求,就发送更新事件。 - - contains_object_list: 仅当 UPDATE 请求携带 Object 列表时才发送更新事件 + - contains_object_list: 仅当 UPDATE 请求携带 Object 列表时才发送更新事件。 """ } } lwm2m_translators { desc { - en: """Topic configuration for LwM2M's gateway publishing and subscription""" - zh: """LwM2M 网关订阅/发布消息的主题映射配置""" + en: """Topic configuration for LwM2M's gateway publishing and subscription.""" + zh: """LwM2M 网关订阅/发布消息的主题映射配置。""" } } @@ -244,14 +244,14 @@ For example, after receiving an update message from a client, any messages withi For each new LwM2M client that succeeds in going online, the gateway creates a subscription relationship to receive downstream commands and send it to the LwM2M client""" zh: """下行命令主题。 -对于每个成功上线的新 LwM2M 客户端,网关会创建一个订阅关系来接收下行消息并将其发送给客户端""" +对于每个成功上线的新 LwM2M 客户端,网关会创建一个订阅关系来接收下行消息并将其发送给客户端。""" } } lwm2m_translators_response { desc { en: """The topic for gateway to publish the acknowledge events from LwM2M client""" - zh: """用于网关发布来自 LwM2M 客户端的确认事件的主题""" + zh: """用于网关发布来自 LwM2M 客户端的确认事件的主题。""" } } @@ -261,28 +261,28 @@ For each new LwM2M client that succeeds in going online, the gateway creates a s After succeed observe a resource of LwM2M client, Gateway will send the notify events via this topic, if the client reports any resource changes""" zh: """用于发布来自 LwM2M 客户端的通知事件的主题。 -在成功 Observe 到 LwM2M 客户端的资源后,如果客户端报告任何资源状态的变化,网关将通过该主题发送通知事件""" +在成功 Observe 到 LwM2M 客户端的资源后,如果客户端报告任何资源状态的变化,网关将通过该主题发送通知事件。""" } } lwm2m_translators_register { desc { en: """The topic for gateway to publish the register events from LwM2M client.""" - zh: """用于发布来自 LwM2M 客户端的注册事件的主题""" + zh: """用于发布来自 LwM2M 客户端的注册事件的主题。""" } } lwm2m_translators_update { desc { en: """The topic for gateway to publish the update events from LwM2M client""" - zh: """用于发布来自LwM2M客户端的更新事件的主题""" + zh: """用于发布来自LwM2M客户端的更新事件的主题。""" } } translator { desc { en: """MQTT topic that corresponds to a particular type of event.""" - zh: """配置某网关客户端对于发布消息或订阅的主题和 QoS 等级""" + zh: """配置某网关客户端对于发布消息或订阅的主题和 QoS 等级。""" } } @@ -412,28 +412,28 @@ After succeed observe a resource of LwM2M client, Gateway will send the notify e gateway_common_authentication { desc { en: """Default authentication configs for all the gateway listeners. For per-listener overrides see authentication\n in listener configs""" - zh: """网关的认证器配置,对该网关下所以的监听器生效。如果每个监听器需要配置不同的认证器,需要配置监听器下的 authentication 字段""" + zh: """网关的认证器配置,对该网关下所以的监听器生效。如果每个监听器需要配置不同的认证器,需要配置监听器下的 authentication 字段。""" } } tcp_udp_listeners { desc { en: """Settings for the listeners.""" - zh: """监听器配置""" + zh: """监听器配置。""" } } tcp_listeners { desc { en: """Settings for the TCP listeners.""" - zh: """配置 TCP 类型的监听器""" + zh: """配置 TCP 类型的监听器。""" } } udp_listeners { desc { en: """Settings for the UDP listeners.""" - zh: """配置 UDP 类型的监听器""" + zh: """配置 UDP 类型的监听器。""" } } @@ -454,7 +454,7 @@ After succeed observe a resource of LwM2M client, Gateway will send the notify e tcp_listener_tcp_opts{ desc { en: """Setting the TCP socket options.""" - zh: """TCP Socket 配置""" + zh: """TCP Socket 配置。""" } } @@ -484,7 +484,7 @@ EMQX will close the TCP connection if proxy protocol packet is not received with ssl_listener_options { desc { en: """SSL Socket options.""" - zh: """SSL Socket 配置""" + zh: """SSL Socket 配置。""" } } @@ -498,7 +498,7 @@ EMQX will close the TCP connection if proxy protocol packet is not received with udp_listener_udp_opts { desc { en: """Settings for the UDP sockets.""" - zh: """UDP Socket 配置 """ + zh: """UDP Socket 配置。""" } } @@ -535,7 +535,7 @@ See: https://erlang.org/doc/man/inet.html#setopts-2""" udp_listener_reuseaddr { desc { en: """Allow local reuse of port numbers.""" - zh: """允许重用本地处于 TIME_WAIT 的端口号""" + zh: """允许重用本地处于 TIME_WAIT 的端口号。""" } } diff --git a/apps/emqx_plugins/i18n/emqx_plugins_schema.conf b/apps/emqx_plugins/i18n/emqx_plugins_schema.conf index 4c2a2fbac..45c02242f 100644 --- a/apps/emqx_plugins/i18n/emqx_plugins_schema.conf +++ b/apps/emqx_plugins/i18n/emqx_plugins_schema.conf @@ -73,7 +73,7 @@ the subdirectory named as emqx_foo_bar-0.1.0. NOTE: For security reasons, this directory should **NOT** be writable by anyone except emqx (or any user which runs EMQX). """ - zh: "插件安装包的目录, 不要自己创建, 只能由emqx用户创建与修改" + zh: "插件安装包的目录,出于安全考虑,该目录应该值允许 emqx,或用于运行 EMQX 服务的用户拥有写入权限。" } label { en: "Install Directory" diff --git a/apps/emqx_psk/i18n/emqx_psk_i18n.conf b/apps/emqx_psk/i18n/emqx_psk_i18n.conf index 3a4e27ca3..6bba9c6d5 100644 --- a/apps/emqx_psk/i18n/emqx_psk_i18n.conf +++ b/apps/emqx_psk/i18n/emqx_psk_i18n.conf @@ -16,7 +16,7 @@ The IDs and secrets can be provided from a file which is configurable by the event_info_client_check_authz_complete() -> event_info_common( 'client.check_authz_complete', - {<<"client check authz complete">>, <<"鉴权结果"/utf8>>}, - {<<"client check authz complete">>, <<"鉴权结果"/utf8>>}, + {<<"client check authz complete">>, <<"授权结果"/utf8>>}, + {<<"client check authz complete">>, <<"授权结果"/utf8>>}, <<"SELECT * FROM \"$events/client_check_authz_complete\"">> ). event_info_session_subscribed() -> diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md index 520ed4578..6471a6d66 100644 --- a/rel/emqx_conf.template.en.md +++ b/rel/emqx_conf.template.en.md @@ -94,14 +94,14 @@ Complex types define data 'boxes' which may contain other complex data or primitive values. There are quite some different primitive types, to name a few: -* `atom()` -* `boolean()` -* `string()` -* `integer()` -* `float()` -* `number()` -* `binary()` # another format of string() -* `emqx_schema:duration()` # time duration, another format of integer() +* `atom()`. +* `boolean()`. +* `string()`. +* `integer()`. +* `float()`. +* `number()`. +* `binary()`, another format of string(). +* `emqx_schema:duration()`, time duration, another format of integer() * ... ::: tip Tip diff --git a/rel/emqx_conf.template.zh.md b/rel/emqx_conf.template.zh.md index 86162a4a9..5cfecda82 100644 --- a/rel/emqx_conf.template.zh.md +++ b/rel/emqx_conf.template.zh.md @@ -1,28 +1,28 @@ -EMQX的配置文件格式是 [HOCON](https://github.com/emqx/hocon) . +EMQX的配置文件格式是 [HOCON](https://github.com/emqx/hocon) 。 HOCON(Human-Optimized Config Object Notation)是一个JSON的超集,非常适用于易于人类读写的配置数据存储。 ## 分层结构 EMQX的配置文件可分为三层,自底向上依次是: -1. 不可变的基础层 `emqx.conf` 加上 `EMQX_` 前缀的环境变量.
+1. 不可变的基础层 `emqx.conf` 加上 `EMQX_` 前缀的环境变量。
修改这一层的配置之后,需要重启节点来使之生效。 1. 集群范围重载层:`$EMQX_NODE__DATA_DIR/configs/cluster-override.conf` 1. 节点本地重载层:`$EMQX_NODE__DATA_DIR/configs/local-override.conf` -如果环境变量 `$EMQX_NODE__DATA_DIR` 没有设置,那么该目录会从 emqx.conf 的 `node.data_dir`配置中读取。 +如果环境变量 `$EMQX_NODE__DATA_DIR` 没有设置,那么该目录会从 `emqx.conf` 的 `node.data_dir` 配置中读取。 配置文件 `cluster-override.conf` 的内容会在运行时被EMQX重写。 这些重写发生在 dashboard UI,管理HTTP API,或者CLI对集群配置进行修改时。 当EMQX运行在集群中时,一个EMQX节点重启之后,会从集群中其他节点复制该文件内容到本地。 :::tip Tip -有些配置项是不能被重载的(例如 `node.name`). +有些配置项是不能被重载的(例如 `node.name`)。 配置项如果有 `mapping: path.to.boot.config.key` 这个属性, -则不能被添加到重载文件中 `*-override.conf` 中。 +则不能被添加到重载文件 `*-override.conf` 中。 ::: -更多的重载规则,请参考下文 [配置重载规则](#配置重载规则). +更多的重载规则,请参考下文 [配置重载规则](#配置重载规则)。 ## 配置文件语法 @@ -70,7 +70,7 @@ EMQX的配置文件中,有4中复杂数据结构类型,它们分别是: 1. Struct:结构体都是有类型名称的,结构体中可以有任意多个字段。 结构体和字段的名称由不带特殊字符的全小些字母组成,名称中可以带数字,但不得以数字开头,多个单词可用下划线分隔。 -1. Map: Map与Struct(结构体)类似,但是内部的字段不是预先定义好的. +1. Map: Map 与 Struct(结构体)类似,但是内部的字段不是预先定义好的。 1. Union: 联合 `MemberType1 | MemberType2 | ...`,可以理解为:“不是这个,就是那个” 1. Array: 数组 `[ElementType]` @@ -89,19 +89,19 @@ myarray.2 = 75 复杂类型定义了数据 "盒子",其中可能包含其他复杂数据或原始值。 有很多不同的原始类型,仅举几个例子。 -* 原子 `atom()` -* 布尔 `boolean()`. -* 字符串 `string()'。 -* 整形 `integer()'。 -* 浮点数 `float()'. -* 数值 `number()'。 -* 二进制编码的字符串 `binary()` # 是 `string()` 的另一种格式 -* 时间间隔 `emqx_schema:duration()` # 时间间隔,是 `integer()` 的另一种格式 +* 原子 `atom()`。 +* 布尔 `boolean()`。 +* 字符串 `string()`。 +* 整形 `integer()`。 +* 浮点数 `float()`。 +* 数值 `number()`。 +* 二进制编码的字符串 `binary()` 是 `string()` 的另一种格式。 +* 时间间隔 `emqx_schema:duration()` 是 `integer()` 的另一种格式。 * ... ::: tip Tip 原始类型的名称大多是自我描述的,所以不需要过多的注释。 -但是有一些不是那么直观的数据类型,则需要配合字段的描述文档进行理解 +但是有一些不是那么直观的数据类型,则需要配合字段的描述文档进行理解。 ::: @@ -110,7 +110,7 @@ myarray.2 = 75 如果我们把EMQX的配置值理解成一个类似目录树的结构,那么类似于文件系统中使用斜杠或反斜杠进行层级分割, EMQX使用的配置路径的层级分割符是 `'.'` -被`'.'`号分割的每一段,则是Struct(结构体)的字段,或Map的key. +被 `'.'` 号分割的每一段,则是 Struct(结构体)的字段,或 Map 的 key。 下面有几个例子: @@ -122,7 +122,7 @@ authentication.1.enable = true ### 环境变量重载 -因为`'.'` 分隔符不能使用于环境变量,所以我们需要使用另一个分割符。EMQX选用的是双下划线`__`。 +因为 `'.'` 分隔符不能使用于环境变量,所以我们需要使用另一个分割符。EMQX选用的是双下划线 `__`。 为了与其他的环境变量有所区分,EMQX还增加了一个前缀 `EMQX_` 来用作环境变量命名空间。 例如 `node.name` 的重载变量名是 `EMQX_NODE__NAME`。 @@ -154,7 +154,7 @@ EMQX_BRIDGES__MQTT__MYBRIDGE__CONNECTOR_SERVER='"localhost:1883"' [warning] unknown_env_vars: ["EMQX_AUTHENTICATION__ENABLED"] ``` -这是因为正确的字段名称是 `enable`,而不是 `enabled`. +这是因为正确的字段名称是 `enable`,而不是 `enabled`。 ::: ### 配置重载规则 @@ -168,8 +168,7 @@ HOCON的值是分层覆盖的,普遍规则如下: #### 结构体 -合并覆盖规则。在如下配置中,最后一行的 `debug` 值会覆盖覆盖原先`level`字段的 `error` 值 -但是`enable` 字段保持不变。 +合并覆盖规则。在如下配置中,最后一行的 `debug` 值会覆盖覆盖原先`level`字段的 `error` 值,但是 `enable` 字段保持不变。 ``` log { console_handler{ @@ -178,7 +177,7 @@ log { } } -## 控制台日志打印先定义为`error`级,后被覆写成`debug`级 +## 控制台日志打印先定义为 `error` 级,后被覆写成 `debug` 级 log.console_handler.level=debug ``` @@ -186,7 +185,7 @@ log.console_handler.level=debug #### Map Map与结构体类似,也是合并覆盖规则。 -如下例子中,`zone1` 的 `max_packet_size` 可以在文件后面覆写. +如下例子中,`zone1` 的 `max_packet_size` 可以在文件后面覆写。 ``` zone { From bb6c3ed4ae42f3a6ba4826c01478cc02aefff8db Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 27 Oct 2022 08:39:58 +0200 Subject: [PATCH 098/165] docs: fix more zh translation desc --- .../i18n/emqx_authz_api_cache_i18n.conf | 2 +- .../i18n/emqx_authz_api_schema_i18n.conf | 28 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/apps/emqx_authz/i18n/emqx_authz_api_cache_i18n.conf b/apps/emqx_authz/i18n/emqx_authz_api_cache_i18n.conf index ea6960555..9c620a22d 100644 --- a/apps/emqx_authz/i18n/emqx_authz_api_cache_i18n.conf +++ b/apps/emqx_authz/i18n/emqx_authz_api_cache_i18n.conf @@ -2,7 +2,7 @@ emqx_authz_api_cache { authorization_cache_delete { desc { en: """Clean all authorization cache in the cluster.""" - zh: """清除集群中所有授权数据缓存""" + zh: """清除集群中所有授权数据缓存。""" } } } diff --git a/apps/emqx_authz/i18n/emqx_authz_api_schema_i18n.conf b/apps/emqx_authz/i18n/emqx_authz_api_schema_i18n.conf index 681cad7a2..afec5c109 100644 --- a/apps/emqx_authz/i18n/emqx_authz_api_schema_i18n.conf +++ b/apps/emqx_authz/i18n/emqx_authz_api_schema_i18n.conf @@ -1,8 +1,8 @@ emqx_authz_api_schema { enable { desc { - en: """Set to true or false to disable this ACL provider""" - zh: """设为 truefalse 以启用或禁用此访问控制数据源""" + en: """Set to true or false to disable this ACL provider.""" + zh: """设为 truefalse 以启用或禁用此访问控制数据源。""" } label { en: """enable""" @@ -13,7 +13,7 @@ emqx_authz_api_schema { type { desc { en: """Backend type.""" - zh: """数据后端类型""" + zh: """数据后端类型。""" } label { en: """type""" @@ -26,7 +26,7 @@ emqx_authz_api_schema { rules { desc { en: """Authorization static file rules.""" - zh: """静态授权文件规则""" + zh: """静态授权文件规则。""" } label { en: """rules""" @@ -39,7 +39,7 @@ emqx_authz_api_schema { method { desc { en: """HTTP method.""" - zh: """HTTP 请求方法""" + zh: """HTTP 请求方法。""" } label { en: """method""" @@ -50,7 +50,7 @@ emqx_authz_api_schema { url { desc { en: """URL of the auth server.""" - zh: """认证服务器 URL""" + zh: """认证服务器 URL。""" } label { en: """url""" @@ -72,7 +72,7 @@ emqx_authz_api_schema { headers_no_content_type { desc { en: """List of HTTP headers (without content-type).""" - zh: """HTTP Headers 列表(无 content-type)""" + zh: """HTTP Headers 列表(无 content-type)。""" } label { en: """headers_no_content_type""" @@ -83,7 +83,7 @@ emqx_authz_api_schema { body { desc { en: """HTTP request body.""" - zh: """HTTP 请求体""" + zh: """HTTP 请求体。""" } label { en: """body""" @@ -94,7 +94,7 @@ emqx_authz_api_schema { request_timeout { desc { en: """Request timeout.""" - zh: """请求超时时间""" + zh: """请求超时时间。""" } label { en: """request_timeout""" @@ -111,7 +111,7 @@ emqx_authz_api_schema { collection { desc { en: """`MongoDB` collection containing the authorization data.""" - zh: """`MongoDB` 授权数据集""" + zh: """`MongoDB` 授权数据集。""" } label { en: """collection""" @@ -153,7 +153,7 @@ Filter supports the following placeholders: cmd { desc { en: """Database query used to retrieve authorization data.""" - zh: """访问控制数据查询命令""" + zh: """访问控制数据查询命令。""" } label { en: """cmd""" @@ -166,7 +166,7 @@ Filter supports the following placeholders: query { desc { en: """Database query used to retrieve authorization data.""" - zh: """访问控制数据查询语句""" + zh: """访问控制数据查询语句。""" } label { en: """query""" @@ -178,8 +178,8 @@ Filter supports the following placeholders: position { desc { - en: """Where to place the source""" - zh: """认证数据源位置""" + en: """Where to place the source.""" + zh: """认证数据源位置。""" } label { en: """position""" From ef5687d465bd52253b5cdf413d88b5d4b47ac9c1 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 25 Oct 2022 16:24:20 +0200 Subject: [PATCH 099/165] fix(emqx_gateway_api): don't crash on unknown status --- .../emqx_dashboard/src/emqx_dashboard.app.src | 2 +- .../src/emqx_dashboard_swagger.erl | 16 ++++++++++- .../test/emqx_swagger_parameter_SUITE.erl | 16 ++++++++++- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_gateway/src/emqx_gateway_api.erl | 28 +++++++++++++------ .../test/emqx_gateway_api_SUITE.erl | 12 ++++++-- changes/v5.0.10-en.md | 2 ++ changes/v5.0.10-zh.md | 2 ++ 8 files changed, 66 insertions(+), 14 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 9d5f85c7b..fb290c551 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.6"}, + {vsn, "5.0.7"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 5674d273c..3904803ac 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -367,11 +367,13 @@ parameters(Params, Module) -> Required = hocon_schema:field_schema(Type, required), Default = hocon_schema:field_schema(Type, default), HoconType = hocon_schema:field_schema(Type, type), + SchemaExtras = hocon_extract_map([enum, default], Type), Meta = init_meta(Default), {ParamType, Refs} = hocon_schema_to_spec(HoconType, Module), + Schema = maps:merge(maps:merge(ParamType, Meta), SchemaExtras), Spec0 = init_prop( [required | ?DEFAULT_FIELDS], - #{schema => maps:merge(ParamType, Meta), name => Name, in => In}, + #{schema => Schema, name => Name, in => In}, Type ), Spec1 = trans_required(Spec0, Required, In), @@ -384,6 +386,18 @@ parameters(Params, Module) -> ), {lists:reverse(SpecList), AllRefs}. +hocon_extract_map(Keys, Type) -> + lists:foldl( + fun(K, M) -> + case hocon_schema:field_schema(Type, K) of + undefined -> M; + V -> M#{K => V} + end + end, + #{}, + Keys + ). + init_meta(undefined) -> #{}; init_meta(Default) -> #{default => Default}. diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index 912459b9d..5dd76acf6 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -6,7 +6,7 @@ -export([paths/0, api_spec/0, schema/1, fields/1]). -export([init_per_suite/1, end_per_suite/1]). -export([t_in_path/1, t_in_query/1, t_in_mix/1, t_without_in/1, t_ref/1, t_public_ref/1]). --export([t_require/1, t_nullable/1, t_method/1, t_api_spec/1]). +-export([t_require/1, t_query_enum/1, t_nullable/1, t_method/1, t_api_spec/1]). -export([t_in_path_trans/1, t_in_query_trans/1, t_in_mix_trans/1, t_ref_trans/1]). -export([t_in_path_trans_error/1, t_in_query_trans_error/1, t_in_mix_trans_error/1]). -export([all/0, suite/0, groups/0]). @@ -30,6 +30,7 @@ groups() -> t_in_mix, t_without_in, t_require, + t_query_enum, t_nullable, t_method, t_public_ref @@ -226,6 +227,17 @@ t_require(_Config) -> validate("/required/false", ExpectSpec), ok. +t_query_enum(_Config) -> + ExpectSpec = [ + #{ + in => query, + name => userid, + schema => #{type => string, enum => [<<"a">>], default => <<"a">>} + } + ], + validate("/query/enum", ExpectSpec), + ok. + t_nullable(_Config) -> NullableFalse = [ #{ @@ -528,6 +540,8 @@ schema("/test/without/in") -> }; schema("/required/false") -> to_schema([{'userid', mk(binary(), #{in => query, required => false})}]); +schema("/query/enum") -> + to_schema([{'userid', mk(binary(), #{in => query, enum => [<<"a">>], default => <<"a">>})}]); schema("/nullable/false") -> to_schema([{'userid', mk(binary(), #{in => query, required => true})}]); schema("/nullable/true") -> diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 98dbd6909..491d0242a 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, grpc, emqx, emqx_authn]}, diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 00d3d4b17..6ae1135b0 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -53,6 +53,8 @@ gateway_insta/2 ]). +-define(KNOWN_GATEWAY_STATUSES, [<<"running">>, <<"stopped">>, <<"unloaded">>]). + %%-------------------------------------------------------------------- %% minirest behaviour callbacks %%-------------------------------------------------------------------- @@ -71,12 +73,22 @@ paths() -> gateway(get, Request) -> Params = maps:get(query_string, Request, #{}), - Status = - case maps:get(<<"status">>, Params, undefined) of - undefined -> all; - S0 -> binary_to_existing_atom(S0, utf8) - end, - {200, emqx_gateway_http:gateways(Status)}; + Status = maps:get(<<"status">>, Params, <<"all">>), + case lists:member(Status, [<<"all">> | ?KNOWN_GATEWAY_STATUSES]) of + true -> + {200, emqx_gateway_http:gateways(binary_to_existing_atom(Status, utf8))}; + false -> + return_http_error( + 400, + [ + "Unknown gateway status in query: ", + Status, + "\n", + "Values allowed: ", + lists:join(", ", ?KNOWN_GATEWAY_STATUSES) + ] + ) + end; gateway(post, Request) -> Body = maps:get(body, Request, #{}), try @@ -241,16 +253,16 @@ params_gateway_name_in_path() -> ]. params_gateway_status_in_qs() -> - %% FIXME: enum in swagger ?? [ {status, mk( binary(), #{ in => query, + enum => ?KNOWN_GATEWAY_STATUSES, required => false, desc => ?DESC(gateway_status_in_qs), - example => <<"">> + example => <<"running">> } )} ]. diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 8532a3a74..29c4fefe7 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -62,8 +62,16 @@ end_per_suite(Conf) -> t_gateway(_) -> {200, Gateways} = request(get, "/gateways"), lists:foreach(fun assert_gw_unloaded/1, Gateways), - {400, BadReq} = request(get, "/gateways/uname_gateway"), - assert_bad_request(BadReq), + {200, UnloadedGateways} = request(get, "/gateways?status=unloaded"), + lists:foreach(fun assert_gw_unloaded/1, UnloadedGateways), + {200, NoRunningGateways} = request(get, "/gateways?status=running"), + ?assertEqual([], NoRunningGateways), + {400, BadReqUnknownGw} = request(get, "/gateways/unknown_gateway"), + assert_bad_request(BadReqUnknownGw), + {400, BadReqInvalidStatus} = request(get, "/gateways?status=invalid_status"), + assert_bad_request(BadReqInvalidStatus), + {400, BadReqUCStatus} = request(get, "/gateways?status=UNLOADED"), + assert_bad_request(BadReqUCStatus), {201, _} = request(post, "/gateways", #{name => <<"stomp">>}), {200, StompGw1} = request(get, "/gateways/stomp"), assert_feilds_apperence( diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index c70ec9977..943ebadba 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -11,3 +11,5 @@ ## Bug fixes - Fix error log message when `mechanism` is missing in authentication config [#8924](https://github.com/emqx/emqx/pull/8924). + +- Fix HTTP 500 issue when unknown `status` parameter is used in `/gateway` API call [#7794](https://github.com/emqx/emqx/pull/9225). diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 26b2737dd..8411a2fb8 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -11,3 +11,5 @@ ## Bug fixes - 优化认认证配置中 `mechanism` 字段缺失情况下的错误日志 [#8924](https://github.com/emqx/emqx/pull/8924)。 + +- 修复未知 `status` 参数导致 `/gateway` API 发生 HTTP 500 错误的问题 [#7794](https://github.com/emqx/emqx/pull/9225) [#7794](https://github.com/emqx/emqx/pull/9225). From c157392452e742ceb024e6e348494f75869a792c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 27 Oct 2022 08:28:18 +0200 Subject: [PATCH 100/165] docs: fix self-closing html tag, change
to
--- apps/emqx/i18n/emqx_limiter_i18n.conf | 2 +- apps/emqx/i18n/emqx_schema_i18n.conf | 161 +++++++++--------- apps/emqx/src/emqx_schema.erl | 6 +- .../i18n/emqx_authz_schema_i18n.conf | 16 +- apps/emqx_bridge/i18n/emqx_bridge_schema.conf | 8 +- .../i18n/emqx_bridge_webhook_schema.conf | 28 +-- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge_schema.erl | 2 +- apps/emqx_conf/i18n/emqx_conf_schema.conf | 156 ++++++++--------- .../i18n/emqx_connector_api.conf | 4 +- .../i18n/emqx_connector_http.conf | 8 +- .../i18n/emqx_connector_mongo.conf | 8 +- .../i18n/emqx_connector_mqtt_schema.conf | 64 +++---- .../i18n/emqx_connector_mysql.conf | 8 +- .../i18n/emqx_connector_pgsql.conf | 8 +- .../i18n/emqx_connector_redis.conf | 8 +- .../i18n/emqx_connector_schema.conf | 4 +- .../emqx_connector/include/emqx_connector.hrl | 7 +- .../emqx_connector/src/emqx_connector.app.src | 2 +- .../src/mqtt/emqx_connector_mqtt_schema.erl | 8 +- .../emqx_dashboard/src/emqx_dashboard.app.src | 2 +- .../src/emqx_dashboard_swagger.erl | 2 +- .../i18n/emqx_gateway_api_authn_i18n.conf | 10 +- .../i18n/emqx_gateway_api_i18n.conf | 12 +- .../i18n/emqx_gateway_api_listeners_i18n.conf | 8 +- .../src/emqx_mgmt_api_configs.erl | 2 +- .../src/emqx_mgmt_api_plugins.erl | 16 +- .../i18n/emqx_plugins_schema.conf | 22 +-- .../src/emqx_retainer_schema.erl | 2 +- rel/emqx_conf.template.en.md | 4 +- rel/emqx_conf.template.zh.md | 2 +- 31 files changed, 298 insertions(+), 294 deletions(-) diff --git a/apps/emqx/i18n/emqx_limiter_i18n.conf b/apps/emqx/i18n/emqx_limiter_i18n.conf index b94750b38..3657df694 100644 --- a/apps/emqx/i18n/emqx_limiter_i18n.conf +++ b/apps/emqx/i18n/emqx_limiter_i18n.conf @@ -113,7 +113,7 @@ the check/consume will succeed, but it will be forced to wait for a short period burst { desc { - en: """The burst, This value is based on rate.
+ en: """The burst, This value is based on rate.
This value + rate = the maximum limit that can be achieved when limiter burst.""" zh: """突发速率。 突发速率允许短时间内速率超过设置的速率值,突发速率 + 速率 = 当前桶能达到的最大速率值""" diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index e798b593d..a1f218bdb 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -107,7 +107,7 @@ emqx_schema { "milliseconds passed since the last time we considered the connection 'congested'.
" "This is to avoid clearing and raising the alarm again too often." zh: "清除警报前的最短时间。
" - "只有当队列中没有挂起的数据,并且连接至少被堵塞了 min_alarm_sustain_duration 毫秒时,
" + "只有当队列中没有挂起的数据,并且连接至少被堵塞了 min_alarm_sustain_duration 毫秒时,
" "报警才会被清除。这是为了避免太频繁地清除和再次发出警报。" } label { @@ -403,7 +403,7 @@ emqx_schema { alarm_actions { desc { - en: "The actions triggered when the alarm is activated.
\n" + en: "The actions triggered when the alarm is activated.
" "Currently, the following actions are supported: log and " "publish.\n" "log is to write the alarm to log (console or file).\n" @@ -411,7 +411,7 @@ emqx_schema { "the system topics:\n" "$SYS/brokers/emqx@xx.xx.xx.x/alarms/activate and\n" "$SYS/brokers/emqx@xx.xx.xx.x/alarms/deactivate" - zh: "警报激活时触发的动作。
\n" + zh: "警报激活时触发的动作。
" "目前,支持以下操作:log 和 " "publish.\n" "log 将告警写入日志 (控制台或者文件).\n" @@ -427,7 +427,7 @@ emqx_schema { alarm_size_limit { desc { - en: "The maximum total number of deactivated alarms to keep as history.
\n" + en: "The maximum total number of deactivated alarms to keep as history.
" "When this limit is exceeded, the oldest deactivated alarms are " "deleted to cap the total number.\n" zh: "要保留为历史记录的已停用报警的最大总数。当超过此限制时,将删除最旧的停用报警,以限制总数。" @@ -1325,11 +1325,11 @@ fields_deflate_opts_level { fields_deflate_opts_mem_level { desc { en: """ -Specifies the size of the compression state.
+Specifies the size of the compression state.
Lower values decrease memory usage per connection. """ zh: """ -指定压缩状态的大小
+指定压缩状态的大小
较低的值会减少每个连接的内存使用。 """ } @@ -1408,19 +1408,19 @@ client_ssl_opts_schema_enable { common_ssl_opts_schema_cacertfile { desc { en: """ -Trusted PEM format CA certificates bundle file.
+Trusted PEM format CA certificates bundle file.
The certificates in this file are used to verify the TLS peer's certificates. Append new certificates to the file if new CAs are to be trusted. There is no need to restart EMQX to have the updated file loaded, because -the system regularly checks if file has been updated (and reload).
+the system regularly checks if file has been updated (and reload).
NOTE: invalidating (deleting) a certificate from the file will not affect already established connections. """ zh: """ -受信任的PEM格式CA证书捆绑文件
+受信任的PEM格式 CA 证书捆绑文件
此文件中的证书用于验证TLS对等方的证书。 -如果要信任新CA,请将新证书附加到文件中。 -无需重启EMQX即可加载更新的文件,因为系统会定期检查文件是否已更新(并重新加载)
+如果要信任新 CA,请将新证书附加到文件中。 +无需重启EMQX即可加载更新的文件,因为系统会定期检查文件是否已更新(并重新加载)
注意:从文件中失效(删除)证书不会影响已建立的连接。 """ } @@ -1433,7 +1433,7 @@ already established connections. common_ssl_opts_schema_certfile { desc { en: """ -PEM format certificates chain file.
+PEM format certificates chain file.
The certificates in this file should be in reversed order of the certificate issue chain. That is, the host's certificate should be placed in the beginning of the file, followed by the immediate issuer certificate and so on. @@ -1441,10 +1441,10 @@ Although the root CA certificate is optional, it should be placed at the end of the file if it is to be added. """ zh: """ -PEM格式证书链文件
-此文件中的证书应与证书颁发链的顺序相反。也就是说,主机的证书应该放在文件的开头,然后是直接颁发者证书,依此类推。 -虽然根CA证书是可选的,但它应该放在 -如果要添加文件,请将其删除。 +PEM格式证书链文件
+此文件中的证书应与证书颁发链的顺序相反。也就是说,主机的证书应该放在文件的开头, +然后是直接颁发者 CA 证书,依此类推,一直到根 CA 证书。 +根 CA 证书是可选的,如果想要添加,应加到文件到最末端。 """ } label: { @@ -1490,10 +1490,15 @@ common_ssl_opts_schema_depth { desc { en: """ Maximum number of non-self-issued intermediate certificates that can follow the peer certificate in a valid certification path. -So, if depth is 0 the PEER must be signed by the trusted ROOT-CA directly; if 1 the path can be PEER, CA, ROOT-CA; if 2 the path can be PEER, CA, CA, ROOT-CA, and so on. The default value is 10. +So, if depth is 0 the PEER must be signed by the trusted ROOT-CA directly;
+if 1 the path can be PEER, Intermediate-CA, ROOT-CA;
+if 2 the path can be PEER, Intermediate-CA1, Intermediate-CA2, ROOT-CA.
""" zh: """ -在有效的证书路径中,可以跟随对等证书的非自颁发中间证书的最大数量。因此,如果深度为0,则对等方必须由受信任的根CA直接签名;如果1,路径可以是PEER、CA、ROOT-CA;如果是2,则路径可以是PEER、CA、CA、ROOT-CA等等。默认值为10。 +在有效的证书路径中,可以跟随对等证书的非自颁发中间证书的最大数量。 +因此,如果深度为0,则对等方必须由受信任的根 CA 直接签名;
+如果是1,路径可以是 PEER、中间 CA、ROOT-CA;
+如果是2,则路径可以是PEER、中间 CA1、中间 CA2、ROOT-CA。 """ } label: { @@ -1522,13 +1527,13 @@ Only used if the private key file is password-protected. common_ssl_opts_schema_versions { desc { en: """ -All TLS/DTLS versions to be supported.
-NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config.
+All TLS/DTLS versions to be supported.
+NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config.
In case PSK cipher suites are intended, make sure to configured ['tlsv1.2', 'tlsv1.1'] here. """ zh: """ -支持所有TLS/DTLS版本
+支持所有TLS/DTLS版本
注:PSK 的 Ciphers 无法在 tlsv1.3 中使用,如果打算使用 PSK 密码套件,请确保这里配置为 ["tlsv1.2","tlsv1.1"]。 """ @@ -1546,7 +1551,7 @@ This config holds TLS cipher suite names separated by comma, or as an array of strings. e.g. "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256" or ["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]. -
+
Ciphers (and their ordering) define the way in which the client and server encrypts information over the network connection. Selecting a good cipher suite is critical for the @@ -1554,47 +1559,47 @@ application's data security, confidentiality and performance. The names should be in OpenSSL string format (not RFC format). All default values and examples provided by EMQX config -documentation are all in OpenSSL format.
+documentation are all in OpenSSL format.
NOTE: Certain cipher suites are only compatible with specific TLS versions ('tlsv1.1', 'tlsv1.2' or 'tlsv1.3') incompatible cipher suites will be silently dropped. For instance, if only 'tlsv1.3' is given in the versions, configuring cipher suites for other versions will have no effect. -
+
-NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config
-If PSK cipher suites are intended, 'tlsv1.3' should be disabled from versions.
+NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config
+If PSK cipher suites are intended, 'tlsv1.3' should be disabled from versions.
PSK cipher suites: "RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, -RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"
+RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"

""" zh: """ 此配置保存由逗号分隔的 TLS 密码套件名称,或作为字符串数组。例如 "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256"["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]。 -
+
密码(及其顺序)定义了客户端和服务器通过网络连接加密信息的方式。 选择一个好的密码套件对于应用程序的数据安全性、机密性和性能至关重要。 名称应为 OpenSSL 字符串格式(而不是 RFC 格式)。 -EMQX 配置文档提供的所有默认值和示例都是 OpenSSL 格式
+EMQX 配置文档提供的所有默认值和示例都是 OpenSSL 格式
注意:某些密码套件仅与特定的 TLS 版本兼容('tlsv1.1'、'tlsv1.2'或'tlsv1.3')。 不兼容的密码套件将被自动删除。 例如,如果只有 versions 仅配置为 tlsv1.3。为其他版本配置密码套件将无效。 -
-注:PSK 的 Ciphers 不支持 tlsv1.3
+
+注:PSK 的 Ciphers 不支持 tlsv1.3
如果打算使用PSK密码套件 tlsv1.3。应在ssl.versions中禁用。 -
+
PSK 密码套件: "RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, -RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"
+RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"

""" } label: { @@ -1610,7 +1615,7 @@ This config holds TLS cipher suite names separated by comma, or as an array of strings. e.g. "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256" or ["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]. -
+
Ciphers (and their ordering) define the way in which the client and server encrypts information over the network connection. Selecting a good cipher suite is critical for the @@ -1618,49 +1623,49 @@ application's data security, confidentiality and performance. The names should be in OpenSSL string format (not RFC format). All default values and examples provided by EMQX config -documentation are all in OpenSSL format.
+documentation are all in OpenSSL format.
NOTE: Certain cipher suites are only compatible with specific TLS versions ('tlsv1.1', 'tlsv1.2' or 'tlsv1.3') incompatible cipher suites will be silently dropped. For instance, if only 'tlsv1.3' is given in the versions, configuring cipher suites for other versions will have no effect. -
+
-NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config
-If PSK cipher suites are intended, 'tlsv1.3' should be disabled from versions.
+NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config
+If PSK cipher suites are intended, 'tlsv1.3' should be disabled from versions.
PSK cipher suites: "RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, -RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"
+RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"
-NOTE: QUIC listener supports only 'tlsv1.3' ciphers
+NOTE: QUIC listener supports only 'tlsv1.3' ciphers
""" zh: """ 此配置保存由逗号分隔的 TLS 密码套件名称,或作为字符串数组。例如 "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256"["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]。 -
+
密码(及其顺序)定义了客户端和服务器通过网络连接加密信息的方式。 选择一个好的密码套件对于应用程序的数据安全性、机密性和性能至关重要。 名称应为 OpenSSL 字符串格式(而不是 RFC 格式)。 -EMQX 配置文档提供的所有默认值和示例都是 OpenSSL 格式
+EMQX 配置文档提供的所有默认值和示例都是 OpenSSL 格式
注意:某些密码套件仅与特定的 TLS 版本兼容('tlsv1.1'、'tlsv1.2'或'tlsv1.3')。 不兼容的密码套件将被自动删除。 例如,如果只有 versions 仅配置为 tlsv1.3。为其他版本配置密码套件将无效。 -
-注:PSK 的 Ciphers 不支持 tlsv1.3
+
+注:PSK 的 Ciphers 不支持 tlsv1.3
如果打算使用PSK密码套件,tlsv1.3。应在ssl.versions中禁用。 -
+
PSK 密码套件: "RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, -RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"
+RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"
注:QUIC 监听器不支持 tlsv1.3 的 ciphers """ @@ -1707,11 +1712,11 @@ server_ssl_opts_schema_dhfile { Path to a file containing PEM-encoded Diffie-Hellman parameters to be used by the server if a cipher suite using Diffie-Hellman key exchange is negotiated. If not specified, default parameters -are used.
+are used.
NOTE: The dhfile option is not supported by TLS 1.3. """ zh: """ -如果协商使用Diffie-Hellman密钥交换的密码套件,则服务器将使用包含PEM编码的Diffie-Hellman参数的文件的路径。如果未指定,则使用默认参数。
+如果协商使用Diffie-Hellman密钥交换的密码套件,则服务器将使用包含PEM编码的Diffie-Hellman参数的文件的路径。如果未指定,则使用默认参数。
注意:TLS 1.3不支持dhfile选项。 """ @@ -1992,8 +1997,8 @@ set to `some_tenant`, then the client actually subscribes to the topic `some_tenant/t`. Similarly, if another client B (connected to the same listener as the client A) sends a message to topic `t`, the message is routed to all the clients subscribed `some_tenant/t`, so client A will receive the -message, with topic name `t`.
-Set to `""` to disable the feature.
+message, with topic name `t`.
+Set to `""` to disable the feature.
Variables in mountpoint string: - ${clientid}: clientid @@ -2004,10 +2009,10 @@ Variables in mountpoint string: 将消息传递给订阅者时,将从主题名称中删除带前缀的字符串。挂载点是一种用户可以用来实现不同侦听器之间消息路由隔离的方法。 -例如,如果客户机 A 使用 listeners.tcp.\.mountpoint 设置为'some_tenant',那么客户端实际上订阅了主题'some_tenant/t'。
-类似地,如果另一个客户端B(与客户端A连接到同一个侦听器)向主题 't' 发送消息,该消息将路由到所有订阅了'some_租户/t'的客户端,因此客户端 A 将接收主题名为't'的消息
+例如,如果客户机 A 使用 listeners.tcp.\.mountpoint 设置为'some_tenant',那么客户端实际上订阅了主题'some_tenant/t'。
+类似地,如果另一个客户端B(与客户端A连接到同一个侦听器)向主题 't' 发送消息,该消息将路由到所有订阅了'some_租户/t'的客户端,因此客户端 A 将接收主题名为't'的消息
-设置为"" 以禁用该功能
+设置为"" 以禁用该功能
mountpoint 字符串中的变量: - ${clientid}: clientid @@ -2070,7 +2075,7 @@ When set to false clients will be allowed to connect without authen mqtt_listener_access_rules { desc { en: """ -The access control rules for this listener.
See: https://github.com/emqtt/esockd#allowdeny +The access control rules for this listener.
See: https://github.com/emqtt/esockd#allowdeny """ zh: """此监听器的访问控制规则。""" } @@ -2083,11 +2088,11 @@ The access control rules for this listener.
See: https://github.com/emqtt/es mqtt_listener_proxy_protocol { desc { en: """ -Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx.
+Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx.
See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ """ zh: """ -如果EMQX集群部署在 HAProxy 或 Nginx 之后,请启用代理协议 V1/2
+如果EMQX集群部署在 HAProxy 或 Nginx 之后,请启用代理协议 V1/2
详情见: https://www.haproxy.com/blog/haproxy/proxy-protocol/ """ } @@ -2150,7 +2155,7 @@ listener_authentication { en: """ Per-listener authentication override. Authentication can be one single authenticator instance or a chain of authenticators as an array. -When authenticating a login (username, client ID, etc.) the authenticators are checked in the configured order.
+When authenticating a login (username, client ID, etc.) the authenticators are checked in the configured order.
""" zh: """ @@ -2208,25 +2213,25 @@ The rate is counted by bytes of packets per second. client_ssl_opts_schema_server_name_indication { desc { en: """ -Specify the host name to be used in TLS Server Name Indication extension.
+Specify the host name to be used in TLS Server Name Indication extension.
For instance, when connecting to "server.example.net", the genuine server which accepts the connection and performs TLS handshake may differ from the host the TLS client initially connects to, e.g. when connecting to an IP address -or when the host has multiple resolvable DNS records
+or when the host has multiple resolvable DNS records
If not specified, it will default to the host name string which is used -to establish the connection, unless it is IP addressed used.
+to establish the connection, unless it is IP addressed used.
The host name is then also used in the host name verification of the peer -certificate.
The special value 'disable' prevents the Server Name +certificate.
The special value 'disable' prevents the Server Name Indication extension from being sent and disables the hostname verification check. """ zh: """ -指定要在 TLS 服务器名称指示扩展中使用的主机名。
+指定要在 TLS 服务器名称指示扩展中使用的主机名。
例如,当连接到 "server.example.net" 时,接受连接并执行 TLS 握手的真正服务器可能与 TLS 客户端最初连接到的主机不同, -例如,当连接到 IP 地址时,或者当主机具有多个可解析的 DNS 记录时
+例如,当连接到 IP 地址时,或者当主机具有多个可解析的 DNS 记录时
如果未指定,它将默认为使用的主机名字符串 -建立连接,除非使用 IP 地址
-然后,主机名也用于对等机的主机名验证证书
+建立连接,除非使用 IP 地址
+然后,主机名也用于对等机的主机名验证证书
特殊值 disable 阻止发送服务器名称指示扩展,并禁用主机名验证检查。 """ } @@ -2239,11 +2244,11 @@ verification check. fields_tcp_opts_active_n { desc { en: """ -Specify the {active, N} option for this Socket.
+Specify the {active, N} option for this Socket.
See: https://erlang.org/doc/man/inet.html#setopts-2 """ zh: """ -为此套接字指定{active,N}选项
+为此套接字指定{active,N}选项
See: https://erlang.org/doc/man/inet.html#setopts-2 """ } @@ -2389,18 +2394,18 @@ The SO_REUSEADDR flag for the connections. fields_trace_payload_encode { desc { en: """ -Determine the format of the payload format in the trace file.
+Determine the format of the payload format in the trace file.
`text`: Text-based protocol or plain text protocol. - It is recommended when payload is JSON encoded.
-`hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.
+ It is recommended when payload is JSON encoded.
+`hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.
`hidden`: payload is obfuscated as `******` """ zh: """ -确定跟踪文件中有效负载格式的格式。
+确定跟踪文件中有效负载格式的格式。
`text`:基于文本的协议或纯文本协议。 -建议在有效负载为JSON编码时使用
-`hex`:二进制十六进制编码。当有效负载是自定义二进制协议时,建议使用此选项
+建议在有效负载为JSON编码时使用
+`hex`:二进制十六进制编码。当有效负载是自定义二进制协议时,建议使用此选项
`hidden`:有效负载被模糊化为 `******` """ } @@ -2445,11 +2450,11 @@ WebSocket消息是否允许包含多个 MQTT 数据包。 fields_ws_opts_compress { desc { en: """ -If true, compress WebSocket messages using zlib.
+If true, compress WebSocket messages using zlib.
The configuration items under deflate_opts belong to the compression-related parameter configuration. """ zh: """ -如果 true,则使用zlib 压缩 WebSocket 消息
+如果 true,则使用zlib 压缩 WebSocket 消息
deflate_opts 下的配置项属于压缩相关参数配置。 """ } @@ -2495,11 +2500,11 @@ fields_ws_opts_fail_if_no_subprotocol { en: """ If true, the server will return an error when the client does not carry the Sec-WebSocket-Protocol field. -
Note: WeChat applet needs to disable this verification. +
Note: WeChat applet needs to disable this verification. """ zh: """ 如果true,当客户端未携带Sec WebSocket Protocol字段时,服务器将返回一个错误。 -
注意:微信小程序需要禁用此验证。 +
注意:微信小程序需要禁用此验证。 """ } label: { @@ -2560,7 +2565,7 @@ If false and check_origin_enable is true, fields_ws_opts_check_origins { desc { en: """ -List of allowed origins.
See check_origin_enable. +List of allowed origins.
See check_origin_enable. """ zh: """ 允许的 origins 列表 diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 17bd03088..74ed51a38 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1698,7 +1698,7 @@ desc("stats") -> desc("authorization") -> "Settings for client authorization."; desc("mqtt") -> - "Global MQTT configuration.
\n" + "Global MQTT configuration.
" "The configs here work as default values which can be overridden\n" "in zone configs"; desc("cache") -> @@ -2115,11 +2115,11 @@ ref(Module, StructName) -> hoconsc:ref(Module, StructName). mk_duration(Desc, OverrideMeta) -> DefaultMeta = #{ desc => Desc ++ - " Time interval is a string that contains a number followed by time unit:
\n" + " Time interval is a string that contains a number followed by time unit:
" "- `ms` for milliseconds,\n" "- `s` for seconds,\n" "- `m` for minutes,\n" - "- `h` for hours;\n
" + "- `h` for hours;\n
" "or combination of whereof: `1h5m0s`" }, hoconsc:mk(typerefl:alias("string", duration()), maps:merge(DefaultMeta, OverrideMeta)). diff --git a/apps/emqx_authz/i18n/emqx_authz_schema_i18n.conf b/apps/emqx_authz/i18n/emqx_authz_schema_i18n.conf index 04118353e..a10128592 100644 --- a/apps/emqx_authz/i18n/emqx_authz_schema_i18n.conf +++ b/apps/emqx_authz/i18n/emqx_authz_schema_i18n.conf @@ -2,37 +2,37 @@ emqx_authz_schema { sources { desc { en: """ -Authorization data sources.
+Authorization data sources.
An array of authorization (ACL) data providers. It is designed as an array, not a hash-map, so the sources can be -ordered to form a chain of access controls.
+ordered to form a chain of access controls.
When authorizing a 'publish' or 'subscribe' action, the configured sources are checked in order. When checking an ACL source, in case the client (identified by username or client ID) is not found, it moves on to the next source. And it stops immediately -once an 'allow' or 'deny' decision is returned.
+once an 'allow' or 'deny' decision is returned.
If the client is not found in any of the sources, -the default action configured in 'authorization.no_match' is applied.
+the default action configured in 'authorization.no_match' is applied.
NOTE: The source elements are identified by their 'type'. It is NOT allowed to configure two or more sources of the same type. """ zh: """ -授权数据源。
+授权数据源。
授权(ACL)数据源的列表。 它被设计为一个数组,而不是一个散列映射, -所以可以作为链式访问控制。
+所以可以作为链式访问控制。
当授权一个 'publish' 或 'subscribe' 行为时, 该配置列表中的所有数据源将按顺序进行检查。 如果在某个客户端未找到时(使用 ClientID 或 Username)。 -将会移动到下一个数据源。直至得到 'allow' 或 'deny' 的结果。
+将会移动到下一个数据源。直至得到 'allow' 或 'deny' 的结果。
如果在任何数据源中都未找到对应的客户端信息。 -配置的默认行为 ('authorization.no_match') 将生效。
+配置的默认行为 ('authorization.no_match') 将生效。
注意: 数据源使用 'type' 进行标识。 diff --git a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf index b1a8e56f9..d64ca3b3a 100644 --- a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf +++ b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf @@ -15,13 +15,13 @@ emqx_bridge_schema { desc { en: """ The ID or the configs of the connector to be used for this bridge. Connector IDs must be of format: -{type}:{name}.
+{type}:{name}.
In config files, you can find the corresponding config entry for a connector by such path: -'connectors.{type}.{name}'.
+'connectors.{type}.{name}'.
""" zh: """ -Bridge 使用的 Connector 的 ID 或者配置。Connector ID 的格式必须为:{type}:{name}.
-在配置文件中,您可以通过以下路径找到 Connector 的相应配置条目:'connector.{type}.{name}'。
""" +Bridge 使用的 Connector 的 ID 或者配置。Connector ID 的格式必须为:{type}:{name}
+在配置文件中,您可以通过以下路径找到 Connector 的相应配置条目:'connector.{type}.{name}'。
""" } label: { en: "Connector ID" diff --git a/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf index fcc817bef..b26c6ffb6 100644 --- a/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf +++ b/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf @@ -25,16 +25,16 @@ emqx_bridge_webhook_schema { config_url { desc { en: """ -The URL of the HTTP Bridge.
+The URL of the HTTP Bridge.
Template with variables is allowed in the path, but variables cannot be used in the scheme, host, -or port part.
+or port part.
For example, http://localhost:9901/${topic} is allowed, but http://${host}:9901/message or http://localhost:${port}/message is not allowed. """ zh: """ -HTTP Bridge 的 URL。
-路径中允许使用带变量的模板,但是 host, port 不允许使用变量模板。
+HTTP Bridge 的 URL。
+路径中允许使用带变量的模板,但是 host, port 不允许使用变量模板。
例如, http://localhost:9901/${topic} 是允许的, 但是 http://${host}:9901/message http://localhost:${port}/message @@ -51,13 +51,13 @@ HTTP Bridge 的 URL。
desc { en: """ The MQTT topic filter to be forwarded to the HTTP server. All MQTT 'PUBLISH' messages with the topic -matching the local_topic will be forwarded.
+matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded. """ zh: """ -发送到 'local_topic' 的消息都会转发到 HTTP 服务器。
+发送到 'local_topic' 的消息都会转发到 HTTP 服务器。
注意:如果这个 Bridge 被用作规则(EMQX 规则引擎)的输出,同时也配置了 'local_topic' ,那么这两部分的消息都会被转发到 HTTP 服务器。 """ } @@ -70,12 +70,12 @@ will be forwarded. config_method { desc { en: """ -The method of the HTTP request. All the available methods are: post, put, get, delete.
-Template with variables is allowed.
+The method of the HTTP request. All the available methods are: post, put, get, delete.
+Template with variables is allowed.
""" zh: """ -HTTP 请求的方法。 所有可用的方法包括:post、put、get、delete。
-允许使用带有变量的模板。
""" +HTTP 请求的方法。 所有可用的方法包括:post、put、get、delete。
+允许使用带有变量的模板。
""" } label: { en: "HTTP Method" @@ -86,11 +86,11 @@ HTTP 请求的方法。 所有可用的方法包括:post、put、get、delete config_headers { desc { en: """ -The headers of the HTTP request.
+The headers of the HTTP request.
Template with variables is allowed. """ zh: """ -HTTP 请求的标头。
+HTTP 请求的标头。
允许使用带有变量的模板。 """ } @@ -103,11 +103,11 @@ HTTP 请求的标头。
config_body { desc { en: """ -The body of the HTTP request.
+The body of the HTTP request.
Template with variables is allowed. """ zh: """ -HTTP 请求的正文。
+HTTP 请求的正文。
允许使用带有变量的模板。""" } label: { diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 8e1c60c29..7890853e4 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "An OTP application"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl index d512df323..e4d0e3d2d 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -94,7 +94,7 @@ direction_field(Dir, Desc) -> #{ required => true, default => egress, - desc => "The direction of the bridge. Can be one of 'ingress' or 'egress'.
" ++ + desc => "The direction of the bridge. Can be one of 'ingress' or 'egress'.
" ++ Desc } )}. diff --git a/apps/emqx_conf/i18n/emqx_conf_schema.conf b/apps/emqx_conf/i18n/emqx_conf_schema.conf index 26f3ac9b5..fc33589ec 100644 --- a/apps/emqx_conf/i18n/emqx_conf_schema.conf +++ b/apps/emqx_conf/i18n/emqx_conf_schema.conf @@ -71,12 +71,12 @@ For more information, see: https://www.erlang.org/doc/man/erl.html desc { en: """Service discovery method for the cluster nodes.""" zh: """集群节点发现方式。可选值为: -- manual: 手动加入集群
-- static: 配置静态节点。配置几个固定的节点,新节点通过连接固定节点中的某一个来加入集群。
-- mcast: 使用 UDP 多播的方式发现节点。
-- dns: 使用 DNS A 记录的方式发现节点。
-- etcd: 使用 etcd 发现节点。
-- k8s: 使用 Kubernetes 发现节点。
+- manual: 手动加入集群
+- static: 配置静态节点。配置几个固定的节点,新节点通过连接固定节点中的某一个来加入集群。
+- mcast: 使用 UDP 多播的方式发现节点。
+- dns: 使用 DNS A 记录的方式发现节点。
+- etcd: 使用 etcd 发现节点。
+- k8s: 使用 Kubernetes 发现节点。
""" } label { @@ -111,9 +111,9 @@ For more information, see: https://www.erlang.org/doc/man/erl.html desc { en: """The Erlang distribution protocol for the cluster.""" zh: """分布式 Erlang 集群协议类型。可选值为: -- inet_tcp: 使用 IPv4
-- inet6_tcp 使用 IPv6
-- inet_tls: 使用 TLS,需要与 node.ssl_dist_optfile 配置一起使用。
+- inet_tcp: 使用 IPv4
+- inet6_tcp 使用 IPv6
+- inet_tls: 使用 TLS,需要与 node.ssl_dist_optfile 配置一起使用。
""" } label { @@ -152,7 +152,7 @@ For more information, see: https://www.erlang.org/doc/man/erl.html cluster_mcast_ports { desc { - en: """List of UDP ports used for service discovery.
+ en: """List of UDP ports used for service discovery.
Note: probe messages are broadcast to all the specified ports. """ zh: """指定多播端口。如有多个端口使用逗号 , 分隔。 @@ -286,7 +286,7 @@ Applicable when cluster.discovery_strategy = dns desc { en: """Key prefix used for EMQX service discovery.""" zh: """指定 etcd 路径的前缀。每个节点在 etcd 中都会创建一个路径: -v2/keys///
+v2/keys///
当 cluster.discovery_strategy 为 etcd 时,此配置项才有效。 """ } @@ -382,7 +382,7 @@ make EMQX to discover IP addresses of peer nodes from Kubernetes API. cluster_k8s_suffix { desc { - en: """Node name suffix.
+ en: """Node name suffix.
Note: this parameter is only relevant when address_type is dns or hostname.""" zh: """当使用 k8s 方式并且 cluster.k8s.address_type 指定为 dns 类型时,可设置 emqx 节点名的后缀。 @@ -426,26 +426,26 @@ belong to different clusters from accidentally connecting to each other.""" node_data_dir { desc { en: """ -Path to the persistent data directory.
-Possible auto-created subdirectories are:
-- `mnesia/`: EMQX's built-in database directory.
-For example, `mnesia/emqx@127.0.0.1`.
-There should be only one such subdirectory.
-Meaning, in case the node is to be renamed (to e.g. `emqx@10.0.1.1`),
-the old dir should be deleted first.
-- `configs`: Generated configs at boot time, and cluster/local override configs.
-- `patches`: Hot-patch beam files are to be placed here.
-- `trace`: Trace log files.
+Path to the persistent data directory.
+Possible auto-created subdirectories are:
+- `mnesia/`: EMQX's built-in database directory.
+For example, `mnesia/emqx@127.0.0.1`.
+There should be only one such subdirectory.
+Meaning, in case the node is to be renamed (to e.g. `emqx@10.0.1.1`),
+the old dir should be deleted first.
+- `configs`: Generated configs at boot time, and cluster/local override configs.
+- `patches`: Hot-patch beam files are to be placed here.
+- `trace`: Trace log files.
**NOTE**: One data dir cannot be shared by two or more EMQX nodes. """ zh: """ -节点数据存放目录,可能会自动创建的子目录如下:
-- `mnesia/`。EMQX的内置数据库目录。例如,`mnesia/emqx@127.0.0.1`。
-如果节点要被重新命名(例如,`emqx@10.0.1.1`)。旧目录应该首先被删除。
-- `configs`。在启动时生成的配置,以及集群/本地覆盖的配置。
-- `patches`: 热补丁文件将被放在这里。
-- `trace`: 日志跟踪文件。
+节点数据存放目录,可能会自动创建的子目录如下:
+- `mnesia/`。EMQX的内置数据库目录。例如,`mnesia/emqx@127.0.0.1`。
+如果节点要被重新命名(例如,`emqx@10.0.1.1`)。旧目录应该首先被删除。
+- `configs`。在启动时生成的配置,以及集群/本地覆盖的配置。
+- `patches`: 热补丁文件将被放在这里。
+- `trace`: 日志跟踪文件。
**注意**: 一个数据dir不能被两个或更多的EMQX节点同时使用。 """ @@ -566,9 +566,9 @@ significant: later configuration files override the previous ones. db_backend { desc { en: """ -Select the backend for the embedded database.
+Select the backend for the embedded database.
rlog is the default backend, -that is suitable for very large clusters.
+that is suitable for very large clusters.
mnesia is a backend that offers decent performance in small clusters. """ zh: """ rlog是默认的数据库,他适用于大规模的集群。 @@ -584,20 +584,20 @@ mnesia是备选数据库,在小集群中提供了很好的性能。 db_role { desc { en: """ -Select a node role.
+Select a node role.
core nodes provide durability of the data, and take care of writes. -It is recommended to place core nodes in different racks or different availability zones.
+It is recommended to place core nodes in different racks or different availability zones.
replicant nodes are ephemeral worker nodes. Removing them from the cluster -doesn't affect database redundancy
-It is recommended to have more replicant nodes than core nodes.
+doesn't affect database redundancy
+It is recommended to have more replicant nodes than core nodes.
Note: this parameter only takes effect when the backend is set to rlog. """ zh: """ -选择节点的角色。
-core 节点提供数据的持久性,并负责写入。建议将核心节点放置在不同的机架或不同的可用区。
-repliant 节点是临时工作节点。 从集群中删除它们,不影响数据库冗余
-建议复制节点多于核心节点。
+选择节点的角色。
+core 节点提供数据的持久性,并负责写入。建议将核心节点放置在不同的机架或不同的可用区。
+repliant 节点是临时工作节点。 从集群中删除它们,不影响数据库冗余
+建议复制节点多于核心节点。
注意:该参数仅在设置backend时生效到 rlog。 """ } @@ -610,17 +610,17 @@ to rlog. db_core_nodes { desc { en: """ -List of core nodes that the replicant will connect to.
+List of core nodes that the replicant will connect to.
Note: this parameter only takes effect when the backend is set -to rlog and the role is set to replicant.
-This value needs to be defined for manual or static cluster discovery mechanisms.
+to rlog and the role is set to replicant.
+This value needs to be defined for manual or static cluster discovery mechanisms.
If an automatic cluster discovery mechanism is being used (such as etcd), there is no need to set this value. """ - zh: """当前节点连接的核心节点列表。
+ zh: """当前节点连接的核心节点列表。
注意:该参数仅在设置backend时生效到 rlog -并且设置rolereplicant时生效。
-该值需要在手动或静态集群发现机制下设置。
+并且设置rolereplicant时生效。
+该值需要在手动或静态集群发现机制下设置。
如果使用了自动集群发现机制(如etcd),则不需要设置该值。 """ } @@ -657,15 +657,15 @@ transaction log entry. db_default_shard_transport { desc { - en: """Defines the default transport for pushing transaction logs.
+ en: """Defines the default transport for pushing transaction logs.
This may be overridden on a per-shard basis in db.shard_transports. gen_rpc uses the gen_rpc library, -distr uses the Erlang distribution.
""" +distr uses the Erlang distribution.
""" zh: """ -定义用于推送事务日志的默认传输。
+定义用于推送事务日志的默认传输。
这可以在 db.shard_transports 中基于每个分片被覆盖。 gen_rpc 使用 gen_rpc 库, -distr 使用 Erlang 发行版。
+distr 使用 Erlang 发行版。
""" } label { @@ -676,13 +676,13 @@ This may be overridden on a per-shard basis in db.shard_transports. db_shard_transports { desc { - en: """Allows to tune the transport method used for transaction log replication, on a per-shard basis.
+ en: """Allows to tune the transport method used for transaction log replication, on a per-shard basis.
gen_rpc uses the gen_rpc library, -distr uses the Erlang distribution.
If not specified, +distr uses the Erlang distribution.
If not specified, the default is to use the value set in db.default_shard_transport.""" - zh: """允许为每个 shard 下的事务日志复制操作的传输方法进行调优。
+ zh: """允许为每个 shard 下的事务日志复制操作的传输方法进行调优。
gen_rpc 使用 gen_rpc 库, -distr 使用 Erlang 自带的 rpc 库。
如果未指定, +distr 使用 Erlang 自带的 rpc 库。
如果未指定, 默认是使用 db.default_shard_transport 中设置的值。 """ } @@ -763,12 +763,12 @@ Ensure that the number of completed transactions is less than the max_hist rpc_port_discovery { desc { - en: """manual: discover ports by tcp_server_port.
+ en: """manual: discover ports by tcp_server_port.
stateless: discover ports in a stateless manner, using the following algorithm. If node name is emqxN@127.0.0.1, where the N is an integer, then the listening port will be 5370 + N.""" zh: """manual: 通过 tcp_server_port 来发现端口。 -
stateless: 使用无状态的方式来发现端口,使用如下算法。如果节点名称是 +
stateless: 使用无状态的方式来发现端口,使用如下算法。如果节点名称是 emqxN@127.0.0.1, N 是一个数字,那么监听端口就是 5370 + N。 """ } @@ -780,9 +780,9 @@ emqxN@127.0.0.1
, N 是一个数字,那么监听端口就是 5370 + N。 rpc_tcp_server_port { desc { - en: """Listening port used by RPC local service.
+ en: """Listening port used by RPC local service.
Note that this config only takes effect when rpc.port_discovery is set to manual.""" - zh: """RPC 本地服务使用的 TCP 端口。
+ zh: """RPC 本地服务使用的 TCP 端口。
只有当 rpc.port_discovery 设置为 manual 时,此配置才会生效。 """ } @@ -794,10 +794,10 @@ Note that this config only takes effect when rpc.port_discovery is set to manual rpc_ssl_server_port { desc { - en: """Listening port used by RPC local service.
+ en: """Listening port used by RPC local service.
Note that this config only takes effect when rpc.port_discovery is set to manual and driver is set to ssl.""" - zh: """RPC 本地服务使用的监听SSL端口。
+ zh: """RPC 本地服务使用的监听SSL端口。
只有当 rpc.port_discovery 设置为 manual 且 dirver 设置为 ssl, 此配置才会生效。 """ @@ -847,9 +847,9 @@ Note that this config only takes effect when rpc.driver is set to < rpc_keyfile { desc { - en: """Path to the private key file for the rpc.certfile.
+ en: """Path to the private key file for the rpc.certfile.
Note: contents of this file are secret, so it's necessary to set permissions to 600.""" - zh: """rpc.certfile 的私钥文件的路径。
+ zh: """rpc.certfile 的私钥文件的路径。
注意:此文件内容是私钥,所以需要设置权限为 600。 """ } @@ -861,9 +861,9 @@ Note: contents of this file are secret, so it's necessary to set permissions to rpc_cacertfile { desc { - en: """Path to certification authority TLS certificate file used to validate rpc.certfile.
+ en: """Path to certification authority TLS certificate file used to validate rpc.certfile.
Note: certificates of all nodes in the cluster must be signed by the same CA.""" - zh: """验证 rpc.certfile 的 CA 证书文件的路径。
+ zh: """验证 rpc.certfile 的 CA 证书文件的路径。
注意:集群中所有节点的证书必须使用同一个 CA 签发。 """ } @@ -1301,17 +1301,17 @@ Supervisor 报告的类型。默认为 error 类型。 authorization { desc { en: """ -Authorization a.k.a. ACL.
-In EMQX, MQTT client access control is extremely flexible.
+Authorization a.k.a. ACL.
+In EMQX, MQTT client access control is extremely flexible.
An out-of-the-box set of authorization data sources are supported. -For example,
-'file' source is to support concise and yet generic ACL rules in a file;
+For example,
+'file' source is to support concise and yet generic ACL rules in a file;
'built_in_database' source can be used to store per-client customizable rule sets, -natively in the EMQX node;
-'http' source to make EMQX call an external HTTP API to make the decision;
-'PostgreSQL' etc. to look up clients or rules from external databases;
+natively in the EMQX node;
+'http' source to make EMQX call an external HTTP API to make the decision;
+'PostgreSQL' etc. to look up clients or rules from external databases;
""" - zh: """ 授权(ACL)。EMQX 支持完整的客户端访问控制(ACL)。
""" + zh: """ 授权(ACL)。EMQX 支持完整的客户端访问控制(ACL)。
""" } label { en: "Authorization" @@ -1321,9 +1321,9 @@ natively in the EMQX node;
desc_cluster { desc { - en: """EMQX nodes can form a cluster to scale up the total capacity.
+ en: """EMQX nodes can form a cluster to scale up the total capacity.
Here holds the configs to instruct how individual nodes can discover each other.""" - zh: """EMQX 节点可以组成一个集群,以提高总容量。
这里指定了节点之间如何连接。""" + zh: """EMQX 节点可以组成一个集群,以提高总容量。
这里指定了节点之间如何连接。""" } label { en: "Cluster" @@ -1422,11 +1422,11 @@ The new node joins the cluster by connecting to one of the bootstrap nodes.""" desc_rpc { desc { - en: """EMQX uses a library called gen_rpc for inter-broker communication.
+ en: """EMQX uses a library called gen_rpc for inter-broker communication.
Most of the time the default config should work, but in case you need to do performance fine-tuning or experiment a bit, this is where to look.""" - zh: """EMQX 使用 gen_rpc 库来实现跨节点通信。
+ zh: """EMQX 使用 gen_rpc 库来实现跨节点通信。
大多数情况下,默认的配置应该可以工作,但如果你需要做一些性能优化或者实验,可以尝试调整这些参数。""" } label { @@ -1472,11 +1472,11 @@ Each sink is represented by a _log handler_, which can be configured independent desc_log_rotation { desc { en: """ -By default, the logs are stored in `./log` directory (for installation from zip file) or in `/var/log/emqx` (for binary installation).
+By default, the logs are stored in `./log` directory (for installation from zip file) or in `/var/log/emqx` (for binary installation).
This section of the configuration controls the number of files kept for each log handler. """ zh: """ -默认情况下,日志存储在 `./log` 目录(用于从 zip 文件安装)或 `/var/log/emqx`(用于二进制安装)。
+默认情况下,日志存储在 `./log` 目录(用于从 zip 文件安装)或 `/var/log/emqx`(用于二进制安装)。
这部分配置,控制每个日志处理进程保留的文件数量。 """ } @@ -1489,11 +1489,11 @@ This section of the configuration controls the number of files kept for each log desc_log_overload_kill { desc { en: """ -Log overload kill features an overload protection that activates when the log handlers use too much memory or have too many buffered log messages.
+Log overload kill features an overload protection that activates when the log handlers use too much memory or have too many buffered log messages.
When the overload is detected, the log handler is terminated and restarted after a cooldown period. """ zh: """ -日志过载终止,具有过载保护功能。当日志处理进程使用过多内存,或者缓存的日志消息过多时该功能被激活。
+日志过载终止,具有过载保护功能。当日志处理进程使用过多内存,或者缓存的日志消息过多时该功能被激活。
检测到过载时,日志处理进程将终止,并在冷却期后重新启动。 """ } diff --git a/apps/emqx_connector/i18n/emqx_connector_api.conf b/apps/emqx_connector/i18n/emqx_connector_api.conf index b79be15fc..2f468fff0 100644 --- a/apps/emqx_connector/i18n/emqx_connector_api.conf +++ b/apps/emqx_connector/i18n/emqx_connector_api.conf @@ -14,11 +14,11 @@ emqx_connector_api { conn_test_post { desc { en: """ -Test creating a new connector by given ID
+Test creating a new connector by given ID
The ID must be of format '{type}:{name}' """ zh: """ -通过给定的 ID 测试创建一个新的连接器
+通过给定的 ID 测试创建一个新的连接器
ID 的格式必须为“{type}:{name}” """ } diff --git a/apps/emqx_connector/i18n/emqx_connector_http.conf b/apps/emqx_connector/i18n/emqx_connector_http.conf index 8664d324f..7583a38ed 100644 --- a/apps/emqx_connector/i18n/emqx_connector_http.conf +++ b/apps/emqx_connector/i18n/emqx_connector_http.conf @@ -2,14 +2,14 @@ emqx_connector_http { base_url { desc { en: """ -The base URL is the URL includes only the scheme, host and port.
+The base URL is the URL includes only the scheme, host and port.
When send an HTTP request, the real URL to be used is the concatenation of the base URL and the -path parameter (passed by the emqx_resource:query/2,3 or provided by the request parameter).
+path parameter (passed by the emqx_resource:query/2,3 or provided by the request parameter).
For example: `http://localhost:9901/` """ zh: """ -base URL 只包含host和port。
-发送HTTP请求时,真实的URL是由base URL 和 path parameter连接而成(通过emqx_resource:query/2,3传递,或者通过请求参数提供)。
+base URL 只包含host和port。
+发送HTTP请求时,真实的URL是由base URL 和 path parameter连接而成(通过emqx_resource:query/2,3传递,或者通过请求参数提供)。
示例:`http://localhost:9901/` """ } diff --git a/apps/emqx_connector/i18n/emqx_connector_mongo.conf b/apps/emqx_connector/i18n/emqx_connector_mongo.conf index e290a8107..e43f7bc33 100644 --- a/apps/emqx_connector/i18n/emqx_connector_mongo.conf +++ b/apps/emqx_connector/i18n/emqx_connector_mongo.conf @@ -47,13 +47,13 @@ emqx_connector_mongo { server { desc { en: """ -The IPv4 or IPv6 address or the hostname to connect to.
-A host entry has the following form: `Host[:Port]`.
+The IPv4 or IPv6 address or the hostname to connect to.
+A host entry has the following form: `Host[:Port]`.
The MongoDB default port 27017 is used if `[:Port]` is not specified. """ zh: """ -将要连接的 IPv4 或 IPv6 地址,或者主机名。
-主机名具有以下形式:`Host[:Port]`。
+将要连接的 IPv4 或 IPv6 地址,或者主机名。
+主机名具有以下形式:`Host[:Port]`。
如果未指定 `[:Port]`,则使用 MongoDB 默认端口 27017。 """ } diff --git a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf b/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf index 99ae420f1..a1268c74b 100644 --- a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf +++ b/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf @@ -2,20 +2,20 @@ emqx_connector_mqtt_schema { mode { desc { en: """ -The mode of the MQTT Bridge.
+The mode of the MQTT Bridge.
-- cluster_shareload: create an MQTT connection on each node in the emqx cluster.
+- cluster_shareload: create an MQTT connection on each node in the emqx cluster.
In 'cluster_shareload' mode, the incoming load from the remote broker is shared by -using shared subscription.
+using shared subscription.
Note that the 'clientid' is suffixed by the node name, this is to avoid clientid conflicts between different nodes. And we can only use shared subscription topic filters for remote_topic of ingress connections. """ zh: """ -MQTT 桥的模式。
+MQTT 桥的模式。
-- cluster_shareload:在 emqx 集群的每个节点上创建一个 MQTT 连接。
-在“cluster_shareload”模式下,来自远程代理的传入负载通过共享订阅的方式接收。
+- cluster_shareload:在 emqx 集群的每个节点上创建一个 MQTT 连接。
+在“cluster_shareload”模式下,来自远程代理的传入负载通过共享订阅的方式接收。
请注意,clientid 以节点名称为后缀,这是为了避免不同节点之间的clientid冲突。 而且对于入口连接的 remote_topic,我们只能使用共享订阅主题过滤器。 """ @@ -135,11 +135,11 @@ broker MUST support this feature. ingress_local_topic { desc { en: """ -Send messages to which topic of the local broker.
+Send messages to which topic of the local broker.
Template with variables is allowed. """ zh: """ -向本地broker的哪个topic发送消息。
+向本地broker的哪个topic发送消息。
允许使用带有变量的模板。 """ } @@ -152,11 +152,11 @@ Template with variables is allowed. ingress_local_qos { desc { en: """ -The QoS of the MQTT message to be sent.
+The QoS of the MQTT message to be sent.
Template with variables is allowed. """ zh: """ -待发送 MQTT 消息的 QoS。
+待发送 MQTT 消息的 QoS。
允许使用带有变量的模板。 """ } @@ -191,11 +191,11 @@ Template with variables is allowed. egress_remote_topic { desc { en: """ -Forward to which topic of the remote broker.
+Forward to which topic of the remote broker.
Template with variables is allowed. """ zh: """ -转发到远程broker的哪个topic。
+转发到远程broker的哪个topic。
允许使用带有变量的模板。 """ } @@ -208,11 +208,11 @@ Template with variables is allowed. egress_remote_qos { desc { en: """ -The QoS of the MQTT message to be sent.
+The QoS of the MQTT message to be sent.
Template with variables is allowed. """ zh: """ -待发送 MQTT 消息的 QoS。
+待发送 MQTT 消息的 QoS。
允许使用带有变量的模板。 """ } @@ -225,11 +225,11 @@ Template with variables is allowed. dir { desc { en: """ -The dir where the replayq file saved.
+The dir where the replayq file saved.
Set to 'false' disables the replayq feature. """ zh: """ -replayq 文件保存的目录。
+replayq 文件保存的目录。
设置为 'false' 会禁用 replayq 功能。 """ } @@ -242,12 +242,12 @@ replayq 文件保存的目录。
seg_bytes { desc { en: """ -The size in bytes of a single segment.
+The size in bytes of a single segment.
A segment is mapping to a file in the replayq dir. If the current segment is full, a new segment (file) will be opened to write. """ zh: """ -单个段的大小(以字节为单位)。
+单个段的大小(以字节为单位)。
一个段映射到 replayq 目录中的一个文件。 如果当前段已满,则新段(文件)将被打开写入。 """ } @@ -260,12 +260,12 @@ A segment is mapping to a file in the replayq dir. If the current segment is ful offload { desc { en: """ -In offload mode, the disk queue is only used to offload queue tail segments.
+In offload mode, the disk queue is only used to offload queue tail segments.
The messages are cached in the memory first, then it writes to the replayq files after the size of the memory cache reaches 'seg_bytes'. """ zh: """ -在Offload模式下,磁盘队列仅用于卸载队列尾段。
+在Offload模式下,磁盘队列仅用于卸载队列尾段。
消息首先缓存在内存中,然后写入replayq文件。内存缓大小为“seg_bytes” 指定的值。 """ } @@ -278,11 +278,11 @@ the memory cache reaches 'seg_bytes'. retain { desc { en: """ -The 'retain' flag of the MQTT message to be sent.
+The 'retain' flag of the MQTT message to be sent.
Template with variables is allowed. """ zh: """ -要发送的 MQTT 消息的“保留”标志。
+要发送的 MQTT 消息的“保留”标志。
允许使用带有变量的模板。 """ } @@ -295,11 +295,11 @@ Template with variables is allowed. payload { desc { en: """ -The payload of the MQTT message to be sent.
+The payload of the MQTT message to be sent.
Template with variables is allowed. """ zh: """ -要发送的 MQTT 消息的负载。
+要发送的 MQTT 消息的负载。
允许使用带有变量的模板。 """ } @@ -323,13 +323,13 @@ Template with variables is allowed. desc_ingress { desc { en: """ -The ingress config defines how this bridge receive messages from the remote MQTT broker, and then send them to the local broker.
-Template with variables is allowed in 'local_topic', 'remote_qos', 'qos', 'retain', 'payload'.
+The ingress config defines how this bridge receive messages from the remote MQTT broker, and then send them to the local broker.
+Template with variables is allowed in 'local_topic', 'remote_qos', 'qos', 'retain', 'payload'.
NOTE: if this bridge is used as the input of a rule (emqx rule engine), and also local_topic is configured, then messages got from the remote broker will be sent to both the 'local_topic' and the rule. """ zh: """ -Ingress 模式定义了这个 bridge 如何从远程 MQTT broker 接收消息,然后将它们发送到本地 broker 。
-允许带有的模板变量: 'local_topic'、'remote_qos'、'qos'、'retain'、'payload' 。
+Ingress 模式定义了这个 bridge 如何从远程 MQTT broker 接收消息,然后将它们发送到本地 broker 。
+允许带有的模板变量: 'local_topic'、'remote_qos'、'qos'、'retain'、'payload' 。
注意:如果这个 bridge 被用作规则的输入(emqx 规则引擎),并且还配置了 local_topic,那么从远程 broker 获取的消息将同时被发送到 'local_topic' 和规则引擎。 """ } @@ -342,13 +342,13 @@ Ingress 模式定义了这个 bridge 如何从远程 MQTT broker 接收消息, desc_egress { desc { en: """ -The egress config defines how this bridge forwards messages from the local broker to the remote broker.
-Template with variables is allowed in 'remote_topic', 'qos', 'retain', 'payload'.
+The egress config defines how this bridge forwards messages from the local broker to the remote broker.
+Template with variables is allowed in 'remote_topic', 'qos', 'retain', 'payload'.
NOTE: if this bridge is used as the action of a rule (emqx rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that matches local_topic will be forwarded. """ zh: """ -Egress 模式定义了 bridge 如何将消息从本地 broker 转发到远程 broker。
-允许带有的模板变量: 'remote_topic'、'qos'、'retain'、'payload' 。
+Egress 模式定义了 bridge 如何将消息从本地 broker 转发到远程 broker。
+允许带有的模板变量: 'remote_topic'、'qos'、'retain'、'payload' 。
注意:如果这个 bridge 作为规则(emqx 规则引擎)的输出,并且还配置了 local_topic,那么从规则引擎中获取的数据和匹配 local_topic 的 MQTT 消息都会被转发到远程 broker 。 """ } diff --git a/apps/emqx_connector/i18n/emqx_connector_mysql.conf b/apps/emqx_connector/i18n/emqx_connector_mysql.conf index 4a5d99254..499caae12 100644 --- a/apps/emqx_connector/i18n/emqx_connector_mysql.conf +++ b/apps/emqx_connector/i18n/emqx_connector_mysql.conf @@ -3,13 +3,13 @@ emqx_connector_mysql { server { desc { en: """ -The IPv4 or IPv6 address or the hostname to connect to.
-A host entry has the following form: `Host[:Port]`.
+The IPv4 or IPv6 address or the hostname to connect to.
+A host entry has the following form: `Host[:Port]`.
The MySQL default port 3306 is used if `[:Port]` is not specified. """ zh: """ -将要连接的 IPv4 或 IPv6 地址,或者主机名。
-主机名具有以下形式:`Host[:Port]`。
+将要连接的 IPv4 或 IPv6 地址,或者主机名。
+主机名具有以下形式:`Host[:Port]`。
如果未指定 `[:Port]`,则使用 MySQL 默认端口 3306。 """ } diff --git a/apps/emqx_connector/i18n/emqx_connector_pgsql.conf b/apps/emqx_connector/i18n/emqx_connector_pgsql.conf index 9731b3e18..6aa792070 100644 --- a/apps/emqx_connector/i18n/emqx_connector_pgsql.conf +++ b/apps/emqx_connector/i18n/emqx_connector_pgsql.conf @@ -3,13 +3,13 @@ emqx_connector_pgsql { server { desc { en: """ -The IPv4 or IPv6 address or the hostname to connect to.
-A host entry has the following form: `Host[:Port]`.
+The IPv4 or IPv6 address or the hostname to connect to.
+A host entry has the following form: `Host[:Port]`.
The PostgreSQL default port 5432 is used if `[:Port]` is not specified. """ zh: """ -将要连接的 IPv4 或 IPv6 地址,或者主机名。
-主机名具有以下形式:`Host[:Port]`。
+将要连接的 IPv4 或 IPv6 地址,或者主机名。
+主机名具有以下形式:`Host[:Port]`。
如果未指定 `[:Port]`,则使用 PostgreSQL 默认端口 5432。 """ } diff --git a/apps/emqx_connector/i18n/emqx_connector_redis.conf b/apps/emqx_connector/i18n/emqx_connector_redis.conf index 0e27ee9c6..228d0805a 100644 --- a/apps/emqx_connector/i18n/emqx_connector_redis.conf +++ b/apps/emqx_connector/i18n/emqx_connector_redis.conf @@ -47,13 +47,13 @@ emqx_connector_redis { server { desc { en: """ -The IPv4 or IPv6 address or the hostname to connect to.
-A host entry has the following form: `Host[:Port]`.
+The IPv4 or IPv6 address or the hostname to connect to.
+A host entry has the following form: `Host[:Port]`.
The Redis default port 6379 is used if `[:Port]` is not specified. """ zh: """ -将要连接的 IPv4 或 IPv6 地址,或者主机名。
-主机名具有以下形式:`Host[:Port]`。
+将要连接的 IPv4 或 IPv6 地址,或者主机名。
+主机名具有以下形式:`Host[:Port]`。
如果未指定 `[:Port]`,则使用 MongoDB 默认端口 27017。 """ } diff --git a/apps/emqx_connector/i18n/emqx_connector_schema.conf b/apps/emqx_connector/i18n/emqx_connector_schema.conf index 1f6fd5381..0a94f5e88 100644 --- a/apps/emqx_connector/i18n/emqx_connector_schema.conf +++ b/apps/emqx_connector/i18n/emqx_connector_schema.conf @@ -14,11 +14,11 @@ emqx_connector_schema { desc_connector { desc { en: """ -Configuration for EMQX connectors.
+Configuration for EMQX connectors.
A connector maintains the data related to the external resources, such as MySQL database. """ zh: """ -EMQX 连接器的配置。
+EMQX 连接器的配置。
连接器维护与外部资源相关的数据,比如 MySQL 数据库。 """ } diff --git a/apps/emqx_connector/include/emqx_connector.hrl b/apps/emqx_connector/include/emqx_connector.hrl index 8945bd14a..52c9929a5 100644 --- a/apps/emqx_connector/include/emqx_connector.hrl +++ b/apps/emqx_connector/include/emqx_connector.hrl @@ -25,14 +25,13 @@ -define(PGSQL_DEFAULT_PORT, 5432). -define(SERVERS_DESC, - "A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].`\n" + "A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].`
" "For each Node should be: " ). -define(SERVER_DESC(TYPE, DEFAULT_PORT), - "\n" - "The IPv4 or IPv6 address or the hostname to connect to.
\n" - "A host entry has the following form: `Host[:Port]`.
\n" + "The IPv4 or IPv6 address or the hostname to connect to.
" + "A host entry has the following form: `Host[:Port]`.
" "The " ++ TYPE ++ " default port " ++ DEFAULT_PORT ++ " is used if `[:Port]` is not specified." ). diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 06da66398..64003209f 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "An OTP application"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index 6fa511d00..018331c8a 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -282,9 +282,9 @@ topic_mappings() -> ingress_desc() -> "\n" "The ingress config defines how this bridge receive messages from the remote MQTT broker, and then\n" - "send them to the local broker.
\n" + "send them to the local broker.
" "Template with variables is allowed in 'local_topic', 'remote_qos', 'qos', 'retain',\n" - "'payload'.
\n" + "'payload'.
" "NOTE: if this bridge is used as the input of a rule (emqx rule engine), and also local_topic is\n" "configured, then messages got from the remote broker will be sent to both the 'local_topic' and\n" "the rule.\n". @@ -292,8 +292,8 @@ ingress_desc() -> egress_desc() -> "\n" "The egress config defines how this bridge forwards messages from the local broker to the remote\n" - "broker.
\n" - "Template with variables is allowed in 'remote_topic', 'qos', 'retain', 'payload'.
\n" + "broker.
" + "Template with variables is allowed in 'remote_topic', 'qos', 'retain', 'payload'.
" "NOTE: if this bridge is used as the action of a rule (emqx rule engine), and also local_topic\n" "is configured, then both the data got from the rule and the MQTT messages that matches\n" "local_topic will be forwarded.\n". diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 9d5f85c7b..fb290c551 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.6"}, + {vsn, "5.0.7"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 5674d273c..d8631b8a2 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -427,7 +427,7 @@ trans_description(Spec, Hocon) -> undefined -> Spec; Desc -> - Desc1 = binary:replace(Desc, [<<"
\n">>, <<"\n">>], <<"
">>, [global]), + Desc1 = binary:replace(Desc, [<<"\n">>], <<"
">>, [global]), Spec#{description => Desc1} end. diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf index b7a65bee1..a9ae33f0c 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf @@ -2,7 +2,7 @@ emqx_gateway_api_authn { get_authn { desc { - en: """Gets the configuration of the specified gateway authenticator.
+ en: """Gets the configuration of the specified gateway authenticator.
Returns 404 when gateway or authentication is not enabled.""" zh: """获取指定网关认证器的配置 当网关或认证未启用时,返回 404。""" @@ -18,11 +18,11 @@ Returns 404 when gateway or authentication is not enabled.""" add_authn { desc { - en: """Enables the authenticator for client authentication for the specified gateway. <\br> -When the authenticator is not configured or turned off, all client connections are assumed to be allowed. <\br> + en: """Enables the authenticator for client authentication for the specified gateway.
+When the authenticator is not configured or turned off, all client connections are assumed to be allowed.
Note: Only one authenticator is allowed to be enabled at a time in the gateway, rather than allowing multiple authenticators to be configured to form an authentication chain as in MQTT.""" - zh: """为指定网关开启认证器实现客户端认证的功能。<\br> -当未配置认证器或关闭认证器时,则认为允许所有客户端的连接。<\br> + zh: """为指定网关开启认证器实现客户端认证的功能。
+当未配置认证器或关闭认证器时,则认为允许所有客户端的连接。
注:在网关中仅支持添加一个认证器,而不是像 MQTT 一样允许配置多个认证器构成认证链。""" } } diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf index f7271e6a0..34e9b8567 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf @@ -32,9 +32,9 @@ including current running status, number of connections, listener status, etc."" update_gateway { desc { - en: """Update the gateway basic configurations and running status.
+ en: """Update the gateway basic configurations and running status.
Note: The Authentication and Listener configurations should be updated by other special APIs. """ - zh: """更新指定网关的基础配置、和启用的状态。
+ zh: """更新指定网关的基础配置、和启用的状态。
注:认证、和监听器的配置更新需参考对应的 API 接口。""" } } @@ -48,10 +48,10 @@ Note: The Authentication and Listener configurations should be updated by other gateway_name_in_qs { desc { - en: """Gateway Name.
+ en: """Gateway Name.
It's enum with `stomp`, `mqttsn`, `coap`, `lwm2m`, `exproto` """ - zh: """网关名称.
+ zh: """网关名称.
可取值为 `stomp`、`mqttsn`、`coap`、`lwm2m`、`exproto` """ } @@ -66,9 +66,9 @@ It's enum with `stomp`, `mqttsn`, `coap`, `lwm2m`, `exproto` gateway_status_in_qs { desc { - en: """Filter gateways by status.
+ en: """Filter gateways by status.
It is enum with `running`, `stopped`, `unloaded`""" - zh: """通过网关状态筛选
+ zh: """通过网关状态筛选
可选值为 `running`、`stopped`、`unloaded`""" } } diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf index 0b7c93afe..dc14a7e01 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf @@ -9,9 +9,9 @@ emqx_gateway_api_listeners { add_listener { desc { - en: """Create the gateway listener.
+ en: """Create the gateway listener.
Note: For listener types not supported by a gateway, this API returns `400: BAD_REQUEST`.""" - zh: """为指定网关添加监听器。
+ zh: """为指定网关添加监听器。
注:对于某网关不支持的监听器类型,该接口会返回 `400: BAD_REQUEST`。""" } } @@ -46,9 +46,9 @@ Note: For listener types not supported by a gateway, this API returns `400: BAD_ add_listener_authn { desc { - en: """Enable authenticator for specified listener for client authentication.
+ en: """Enable authenticator for specified listener for client authentication.
When authenticator is enabled for a listener, all clients connecting to that listener will use that authenticator for authentication.""" - zh: """为指定监听器开启认证器以实现客户端认证的能力。
+ zh: """为指定监听器开启认证器以实现客户端认证的能力。
当某一监听器开启认证后,所有连接到该监听器的客户端会使用该认证器进行认证。""" } } diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 8eb801952..e38b6b729 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -115,7 +115,7 @@ schema("/configs_reset/:rootname") -> tags => ?TAGS, description => << - "Reset the config entry specified by the query string parameter `conf_path`.
\n" + "Reset the config entry specified by the query string parameter `conf_path`.
" "- For a config entry that has default value, this resets it to the default value;\n" "- For a config entry that has no default value, an error 400 will be returned" >>, diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index 88dd21518..ac9c8644d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -70,8 +70,8 @@ schema("/plugins") -> 'operationId' => list_plugins, get => #{ description => - "List all install plugins.
" - "Plugins are launched in top-down order.
" + "List all install plugins.
" + "Plugins are launched in top-down order.
" "Using `POST /plugins/{name}/move` to change the boot order.", tags => ?TAGS, responses => #{ @@ -136,9 +136,9 @@ schema("/plugins/:name/:action") -> 'operationId' => update_plugin, put => #{ description => - "start/stop a installed plugin.
" - "- **start**: start the plugin.
" - "- **stop**: stop the plugin.
", + "start/stop a installed plugin.
" + "- **start**: start the plugin.
" + "- **stop**: stop the plugin.
", tags => ?TAGS, parameters => [ hoconsc:ref(name), @@ -272,9 +272,9 @@ fields(running_status) -> {status, hoconsc:mk(hoconsc:enum([running, stopped]), #{ desc => - "Install plugin status at runtime
" - "1. running: plugin is running.
" - "2. stopped: plugin is stopped.
" + "Install plugin status at runtime
" + "1. running: plugin is running.
" + "2. stopped: plugin is stopped.
" })} ]. diff --git a/apps/emqx_plugins/i18n/emqx_plugins_schema.conf b/apps/emqx_plugins/i18n/emqx_plugins_schema.conf index 45c02242f..454d36f6f 100644 --- a/apps/emqx_plugins/i18n/emqx_plugins_schema.conf +++ b/apps/emqx_plugins/i18n/emqx_plugins_schema.conf @@ -2,14 +2,14 @@ emqx_plugins_schema { plugins { desc { en: """ -Manage EMQX plugins.
+Manage EMQX plugins.
Plugins can be pre-built as a part of EMQX package, or installed as a standalone package in a location specified by -install_dir config key
+install_dir config key
The standalone-installed plugins are referred to as 'external' plugins. """ - zh: """管理EMQX插件。
-插件可以是EMQX安装包中的一部分,也可以是一个独立的安装包。
+ zh: """管理EMQX插件。
+插件可以是EMQX安装包中的一部分,也可以是一个独立的安装包。
独立安装的插件称为“外部插件”。 """ } @@ -30,11 +30,11 @@ The standalone-installed plugins are referred to as 'external' plugins. } name_vsn { desc { - en: """The {name}-{version} of the plugin.
-It should match the plugin application name-version as the for the plugin release package name
+ en: """The {name}-{version} of the plugin.
+It should match the plugin application name-version as the for the plugin release package name
For example: my_plugin-0.1.0. """ - zh: """插件的名称{name}-{version}。
+ zh: """插件的名称{name}-{version}。
它应该与插件的发布包名称一致,如my_plugin-0.1.0。""" } label { @@ -54,7 +54,7 @@ For example: my_plugin-0.1.0. } states { desc { - en: """An array of plugins in the desired states.
+ en: """An array of plugins in the desired states.
The plugins are started in the defined order""" zh: """一组插件的状态。插件将按照定义的顺序启动""" } @@ -69,7 +69,7 @@ The plugins are started in the defined order""" The installation directory for the external plugins. The plugin beam files and configuration files should reside in the subdirectory named as emqx_foo_bar-0.1.0. -
+
NOTE: For security reasons, this directory should **NOT** be writable by anyone except emqx (or any user which runs EMQX). """ @@ -82,10 +82,10 @@ by anyone except emqx (or any user which runs EMQX). } check_interval { desc { - en: """Check interval: check if the status of the plugins in the cluster is consistent,
+ en: """Check interval: check if the status of the plugins in the cluster is consistent,
if the results of 3 consecutive checks are not consistent, then alarm. """ - zh: """检查间隔:检查集群中插件的状态是否一致,
+ zh: """检查间隔:检查集群中插件的状态是否一致,
如果连续3次检查结果不一致,则报警。 """ } diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 51dbf496b..25fd27538 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -119,7 +119,7 @@ retainer_indices(type) -> retainer_indices(desc) -> "Retainer index specifications: list of arrays of positive ascending integers. " "Each array specifies an index. Numbers in an index specification are 1-based " - "word positions in topics. Words from specified positions will be used for indexing.
" + "word positions in topics. Words from specified positions will be used for indexing.
" "For example, it is good to have [2, 4] index to optimize " "+/X/+/Y/... topic wildcard subscriptions."; retainer_indices(example) -> diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md index 6471a6d66..0b40178c3 100644 --- a/rel/emqx_conf.template.en.md +++ b/rel/emqx_conf.template.en.md @@ -1,4 +1,4 @@ -EMQX configuration file is in [HOCON](https://github.com/emqx/hocon) format. +EMQX configuration files are in [HOCON](https://github.com/emqx/hocon) format. HOCON, or Human-Optimized Config Object Notation is a format for human-readable data, and a superset of JSON. @@ -7,7 +7,7 @@ and a superset of JSON. EMQX configuration consists of 3 layers. From bottom up: -1. Immutable base: `emqx.conf` + `EMQX_` prefixed environment variables.
+1. Immutable base: `emqx.conf` + `EMQX_` prefixed environment variables.
Changes in this layer require a full node restart to take effect. 1. Cluster overrides: `$EMQX_NODE__DATA_DIR/configs/cluster-override.conf` 1. Local node overrides: `$EMQX_NODE__DATA_DIR/configs/local-override.conf` diff --git a/rel/emqx_conf.template.zh.md b/rel/emqx_conf.template.zh.md index 5cfecda82..9b430da36 100644 --- a/rel/emqx_conf.template.zh.md +++ b/rel/emqx_conf.template.zh.md @@ -5,7 +5,7 @@ HOCON(Human-Optimized Config Object Notation)是一个JSON的超集,非常 EMQX的配置文件可分为三层,自底向上依次是: -1. 不可变的基础层 `emqx.conf` 加上 `EMQX_` 前缀的环境变量。
+1. 不可变的基础层 `emqx.conf` 加上 `EMQX_` 前缀的环境变量。
修改这一层的配置之后,需要重启节点来使之生效。 1. 集群范围重载层:`$EMQX_NODE__DATA_DIR/configs/cluster-override.conf` 1. 节点本地重载层:`$EMQX_NODE__DATA_DIR/configs/local-override.conf` From 2b3748304d609a8853351c65234bf8a20b3fc148 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 27 Oct 2022 15:27:38 +0200 Subject: [PATCH 101/165] chore: bump jq to v0.3.8 (remove -flat_namespace flag on Darwin) --- rebar.config.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.erl b/rebar.config.erl index eb79e6028..85130fb45 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -42,7 +42,7 @@ quicer() -> {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.16"}}}. jq() -> - {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.6"}}}. + {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.8"}}}. deps(Config) -> {deps, OldDeps} = lists:keyfind(deps, 1, Config), From dbc9ae27c54b658a4d5d8fa8538bbe49009d6e1f Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 27 Oct 2022 15:44:32 +0200 Subject: [PATCH 102/165] chore: update jq version in mix --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index de827bc07..0f0d67803 100644 --- a/mix.exs +++ b/mix.exs @@ -616,7 +616,7 @@ defmodule EMQXUmbrella.MixProject do defp jq_dep() do if enable_jq?(), - do: [{:jq, github: "emqx/jq", tag: "v0.3.6", override: true}], + do: [{:jq, github: "emqx/jq", tag: "v0.3.8", override: true}], else: [] end From d4adf8532ea9ee885842aa2c62c20ae0aa07650f Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 27 Oct 2022 18:14:46 +0200 Subject: [PATCH 103/165] fix: typo in http error code --- apps/emqx_dashboard/src/emqx_dashboard.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 6c2a02e47..e032eb28b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -235,7 +235,7 @@ authorize(Req) -> ) end; {error, _} -> - return_unauthorized(<<"WORNG_USERNAME_OR_PWD">>, <<"Check username/password">>) + return_unauthorized(?WRONG_USERNAME_OR_PWD, <<"Check username/password">>) end; {bearer, Token} -> case emqx_dashboard_admin:verify_token(Token) of From d0ca79fc9eb69f57ea3b7f20531f89989b05e41e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 25 Oct 2022 16:16:20 -0300 Subject: [PATCH 104/165] feat(mgmt): add alias endpoint for health check This allow use to reference `/status` in the API documentation without manually editing the `swagger.json` file. --- .../i18n/emqx_mgmt_api_status_i18n.conf | 38 +++++++++++++++++ .../src/emqx_mgmt_api_status.erl | 42 +++++++++++++++++++ .../test/emqx_mgmt_api_status_SUITE.erl | 40 +++++++++++++++--- 3 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf new file mode 100644 index 000000000..9c09806cd --- /dev/null +++ b/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf @@ -0,0 +1,38 @@ +emqx_mgmt_api_status { + get_status_api { + desc { + en: "Serves as a health check for the node. Returns a plain text response" + " describing the status of the node. This endpoint requires no" + " authentication.\n" + "\n" + "Returns status code 200 if the EMQX application is up and running, " + "503 otherwise." + "\n" + "The GET `/status` endpoint (without the `/api/...` prefix) is also an alias" + " to this endpoint and works in the same way." + zh: "作为节点的健康检查。 返回一个纯文本的响应,描述节点的状态。\n" + "\n" + "如果 EMQX 应用程序已经启动并运行,返回状态代码 200,否则返回 503。\n" + "\n" + "GET `/status`端点(没有`/api/...`前缀)也是这个端点的一个别名,工作方式相同。" + } + } + + get_status_response200 { + desc { + en: "Node emqx@127.0.0.1 is started\n" + "emqx is running" + zh: "Node emqx@127.0.0.1 is started\n" + "emqx is running" + } + } + + get_status_response503 { + desc { + en: "Node emqx@127.0.0.1 is stopped\n" + "emqx is not_running" + zh: "Node emqx@127.0.0.1 is stopped\n" + "emqx is not_running" + } + } +} diff --git a/apps/emqx_management/src/emqx_mgmt_api_status.erl b/apps/emqx_management/src/emqx_mgmt_api_status.erl index b313f4bcc..ea91f1c03 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_status.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_status.erl @@ -15,11 +15,50 @@ %%-------------------------------------------------------------------- -module(emqx_mgmt_api_status). +-behaviour(minirest_api). + +-include_lib("hocon/include/hoconsc.hrl"). + +%% minirest API +-export([api_spec/0, paths/0, schema/1]). + +-export([get_status/2]). + -export([ init/2, path/0 ]). +-define(TAGS, [<<"Status">>]). + +%%-------------------------------------------------------------------- +%% minirest API and schema +%%-------------------------------------------------------------------- + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + ["/status"]. + +schema("/status") -> + #{ + 'operationId' => get_status, + get => #{ + description => ?DESC(get_status_api), + tags => ?TAGS, + security => [], + responses => #{ + 200 => ?DESC(get_status_response200), + 503 => ?DESC(get_status_response503) + } + } + }. + +%%-------------------------------------------------------------------- +%% non-minirest (cowboy) API +%%-------------------------------------------------------------------- + %% Note: Because swagger now requires an HTTP prefix (e.g. /api/v5), %% but the `/status` does not require this fixed prefix. %% @@ -39,6 +78,9 @@ init(Req0, State) -> %% API Handler funcs %%-------------------------------------------------------------------- +get_status(get, _Params) -> + running_status(). + running_status() -> case emqx_dashboard_listener:is_ready(timer:seconds(20)) of true -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl index 42d833bda..a768d2bfe 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl @@ -19,6 +19,7 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). -define(HOST, "http://127.0.0.1:18083/"). @@ -27,7 +28,24 @@ %%--------------------------------------------------------------------------------------- all() -> - emqx_common_test_helpers:all(?MODULE). + OtherTCs = emqx_common_test_helpers:all(?MODULE) -- get_status_tests(), + [ + {group, api_status_endpoint}, + {group, non_api_status_endpoint} + | OtherTCs + ]. + +get_status_tests() -> + [ + t_status_ok, + t_status_not_ok + ]. + +groups() -> + [ + {api_status_endpoint, [], get_status_tests()}, + {non_api_status_endpoint, [], get_status_tests()} + ]. init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite(), @@ -36,6 +54,16 @@ init_per_suite(Config) -> end_per_suite(_) -> emqx_mgmt_api_test_util:end_suite(). +init_per_group(api_status_endpoint, Config) -> + [{get_status_path, ["api", "v5", "status"]} | Config]; +init_per_group(non_api_status_endpoint, Config) -> + [{get_status_path, ["status"]} | Config]; +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + init_per_testcase(t_status_not_ok, Config) -> ok = application:stop(emqx), Config; @@ -97,13 +125,14 @@ data_loop(Gun, Ref, Acc) -> %% Test cases %%--------------------------------------------------------------------------------------- -t_status_ok(_Config) -> +t_status_ok(Config) -> + Path = ?config(get_status_path, Config), #{ body := Resp, status_code := StatusCode } = do_request(#{ method => get, - path => ["status"], + path => Path, headers => [], body => no_body }), @@ -114,14 +143,15 @@ t_status_ok(_Config) -> ), ok. -t_status_not_ok(_Config) -> +t_status_not_ok(Config) -> + Path = ?config(get_status_path, Config), #{ body := Resp, headers := Headers, status_code := StatusCode } = do_request(#{ method => get, - path => ["status"], + path => Path, headers => [], body => no_body }), From 554c4b4c3bd597682836a35c6a6bff553ede372d Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 27 Oct 2022 19:04:13 +0200 Subject: [PATCH 105/165] build: stop trying to fetch rebar and build deps on each build --- .gitignore | 1 + Makefile | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index d8b3806e3..d01c764d0 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,4 @@ apps/emqx/test/emqx_static_checks_data/master.bpapi # rendered configurations *.conf.rendered lux_logs/ +/.prepare diff --git a/Makefile b/Makefile index eeabaed33..e9238d098 100644 --- a/Makefile +++ b/Makefile @@ -30,12 +30,10 @@ export REBAR_GIT_CLONE_OPTIONS += --depth=1 .PHONY: default default: $(REBAR) $(PROFILE) -.PHONY: prepare -prepare: FORCE +.prepare: @$(SCRIPTS)/git-hooks-init.sh # this is no longer needed since 5.0 but we keep it anyway @$(SCRIPTS)/prepare-build-deps.sh - -FORCE: + @touch .prepare .PHONY: all all: $(REBAR) $(PROFILES) @@ -44,6 +42,9 @@ all: $(REBAR) $(PROFILES) ensure-rebar3: @$(SCRIPTS)/ensure-rebar3.sh +$(REBAR): .prepare + @$(SCRIPTS)/ensure-rebar3.sh + .PHONY: ensure-hex ensure-hex: @mix local.hex --if-missing --force @@ -60,8 +61,6 @@ ensure-mix-rebar: $(REBAR) mix-deps-get: $(ELIXIR_COMMON_DEPS) @mix deps.get -$(REBAR): prepare ensure-rebar3 - .PHONY: eunit eunit: $(REBAR) conf-segs @ENABLE_COVER_COMPILE=1 $(REBAR) eunit -v -c --cover_export_name $(PROFILE)-eunit From dcab22458d2eeb50a683683c9e0023dfcb57ae9a Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 27 Oct 2022 19:41:57 +0200 Subject: [PATCH 106/165] build: always ensure rebar3 --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e9238d098..7644e1e2a 100644 --- a/Makefile +++ b/Makefile @@ -42,8 +42,7 @@ all: $(REBAR) $(PROFILES) ensure-rebar3: @$(SCRIPTS)/ensure-rebar3.sh -$(REBAR): .prepare - @$(SCRIPTS)/ensure-rebar3.sh +$(REBAR): .prepare ensure-rebar3 .PHONY: ensure-hex ensure-hex: From 4cadb86b18d0c404509e848b4b2b6aa0ef1b8563 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 27 Oct 2022 20:09:31 +0200 Subject: [PATCH 107/165] docs: fix typos --- apps/emqx/i18n/emqx_schema_i18n.conf | 2 +- rel/emqx_conf.template.en.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index a1f218bdb..23784785a 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1529,7 +1529,7 @@ common_ssl_opts_schema_versions { en: """ All TLS/DTLS versions to be supported.
NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config.
-In case PSK cipher suites are intended, make sure to configured +In case PSK cipher suites are intended, make sure to configure ['tlsv1.2', 'tlsv1.1'] here. """ zh: """ diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md index 0b40178c3..46ff2b36b 100644 --- a/rel/emqx_conf.template.en.md +++ b/rel/emqx_conf.template.en.md @@ -146,7 +146,7 @@ For example, this environment variable sets an array value. export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]' ``` -However this also means a string value should be quoted if it happen to contain special +However this also means a string value should be quoted if it happens to contain special characters such as `=` and `:`. For example, a string value `"localhost:1883"` would be @@ -248,9 +248,9 @@ authentication=[{enable=true}] #### TLS/SSL ciphers -Starting from v5.0.6, EMQX no longer pre-populate the ciphers list with a default +Starting from v5.0.6, EMQX no longer pre-populates the ciphers list with a default set of cipher suite names. -Instead, the default ciphers are applyed at runtime when starting the listener +Instead, the default ciphers are applied at runtime when starting the listener for servers, or when establishing a TLS connection as a client. Below are the default ciphers selected by EMQX. From d704de4a84ceaab217403b952d839bdefc6ab183 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 25 Oct 2022 19:32:37 +0200 Subject: [PATCH 108/165] fix(emqx_rule_events): drop tuple-value from message headers the message headers are fed to a JSON ecnoder which may crash if the header value is a tuple --- apps/emqx_rule_engine/src/emqx_rule_events.erl | 7 +++++-- .../emqx_rule_engine/test/emqx_rule_events_SUITE.erl | 12 +++++++++--- changes/v5.0.10-en.md | 5 +++++ changes/v5.0.10-zh.md | 4 ++++ 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index b4a7c34b8..2935aeeb9 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -1054,8 +1054,11 @@ printable_maps(Headers) -> || {Key, Value} <- V0 ] }; - (K, V0, AccIn) -> - AccIn#{K => V0} + (_K, V, AccIn) when is_tuple(V) -> + %% internal headers + AccIn; + (K, V, AccIn) -> + AccIn#{K => V} end, #{}, Headers diff --git a/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl index ad8d28159..c9774b93d 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl @@ -26,13 +26,19 @@ t_printable_maps(_) -> Headers = #{ peerhost => {127, 0, 0, 1}, peername => {{127, 0, 0, 1}, 9980}, - sockname => {{127, 0, 0, 1}, 1883} + sockname => {{127, 0, 0, 1}, 1883}, + redispatch_to => {<<"group">>, <<"sub/topic/+">>}, + shared_dispatch_ack => {self(), ref} }, + Converted = emqx_rule_events:printable_maps(Headers), ?assertMatch( #{ peerhost := <<"127.0.0.1">>, peername := <<"127.0.0.1:9980">>, sockname := <<"127.0.0.1:1883">> }, - emqx_rule_events:printable_maps(Headers) - ). + Converted + ), + ?assertNot(maps:is_key(redispatch_to, Converted)), + ?assertNot(maps:is_key(shared_dispatch_ack, Converted)), + ok. diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index 667431852..2cca0103b 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -19,3 +19,8 @@ - Fixed the HTTP response status code for the `/status` endpoint [#9211](https://github.com/emqx/emqx/pull/9211). Before the fix, it always returned `200` even if the EMQX application was not running. Now it returns `503` in that case. + +- Fix message delivery related event encoding [#9228](https://github.com/emqx/emqx/pull/9228). + This bug was introduced in v5.0.9. For Rule-Engine's input events like `$events/message_delivered` + and `$events/message_dropped`, if the message was delivered to a shared-subscription, + the encoding (to JSON) of the event will fail. diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index b32c21ba6..6f6b75e02 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -18,3 +18,7 @@ - 修正了 `/status` 端点的响应状态代码 [#9211](https://github.com/emqx/emqx/pull/9211)。 在此修复前,它总是返回 HTTP 状态码 `200`,即使 EMQX 没有完成启动或正在重启。 现在它在这些情况下会返回状态码 `503`。 + +- 修复规则引擎的消息事件编码失败 [#9228](https://github.com/emqx/emqx/pull/9228)。 + 该问题在 v5.0.9 中引入:带消息的规则引擎事件,例如 `$events/message_delivered` 和 + `$events/message_dropped`, 如果消息事件是共享订阅产生的,在编码(到 JSON 格式)过程中会失败。 From ea95f792e086edd8e19ac6fa937bb7e6b00d911f Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 25 Oct 2022 07:08:30 +0800 Subject: [PATCH 109/165] refactor: multicall node_info to improve /node speed --- apps/emqx/src/emqx_rpc.erl | 3 + apps/emqx_management/src/emqx_mgmt.erl | 63 ++++++++------- .../src/proto/emqx_management_proto_v3.erl | 80 +++++++++++++++++++ .../test/emqx_mgmt_api_nodes_SUITE.erl | 52 ++++++++++++ 4 files changed, 168 insertions(+), 30 deletions(-) create mode 100644 apps/emqx_management/src/proto/emqx_management_proto_v3.erl diff --git a/apps/emqx/src/emqx_rpc.erl b/apps/emqx/src/emqx_rpc.erl index 939b5395d..ec8fd83de 100644 --- a/apps/emqx/src/emqx_rpc.erl +++ b/apps/emqx/src/emqx_rpc.erl @@ -125,6 +125,9 @@ max_client_num() -> emqx:get_config([rpc, tcp_client_num], ?DefaultClientNum). -spec unwrap_erpc(emqx_rpc:erpc(A)) -> A | {error, _Err}. + +unwrap_erpc(Res) when is_list(Res) -> + [unwrap_erpc(R) || R <- Res]; unwrap_erpc({ok, A}) -> A; unwrap_erpc({throw, A}) -> diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 8f73d5767..23804614b 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -118,9 +118,11 @@ list_nodes() -> Running = mria_mnesia:cluster_nodes(running), Stopped = mria_mnesia:cluster_nodes(stopped), DownNodes = lists:map(fun stopped_node_info/1, Stopped), - [{Node, node_info(Node)} || Node <- Running] ++ DownNodes. + [{Node, Info} || #{node := Node} = Info <- node_info(Running)] ++ DownNodes. -lookup_node(Node) -> node_info(Node). +lookup_node(Node) -> + [Info] = node_info([Node]), + Info. node_info() -> {UsedRatio, Total} = get_sys_memory(), @@ -152,8 +154,8 @@ get_sys_memory() -> {0, 0} end. -node_info(Node) -> - wrap_rpc(emqx_management_proto_v2:node_info(Node)). +node_info(Nodes) -> + emqx_rpc:unwrap_erpc(emqx_management_proto_v3:node_info(Nodes)). stopped_node_info(Node) -> #{name => Node, node_status => 'stopped'}. @@ -163,17 +165,19 @@ stopped_node_info(Node) -> %%-------------------------------------------------------------------- list_brokers() -> - [{Node, broker_info(Node)} || Node <- mria_mnesia:running_nodes()]. + Running = mria_mnesia:running_nodes(), + [{Node, Broker} || #{node := Node} = Broker <- broker_info(Running)]. lookup_broker(Node) -> - broker_info(Node). + [Broker] = broker_info([Node]), + Broker. broker_info() -> Info = maps:from_list([{K, iolist_to_binary(V)} || {K, V} <- emqx_sys:info()]), Info#{node => node(), otp_release => otp_rel(), node_status => 'Running'}. -broker_info(Node) -> - wrap_rpc(emqx_management_proto_v2:broker_info(Node)). +broker_info(Nodes) -> + emqx_rpc:unwrap_erpc(emqx_management_proto_v3:broker_info(Nodes)). %%-------------------------------------------------------------------- %% Metrics and Stats @@ -183,7 +187,7 @@ get_metrics() -> nodes_info_count([get_metrics(Node) || Node <- mria_mnesia:running_nodes()]). get_metrics(Node) -> - wrap_rpc(emqx_proto_v1:get_metrics(Node)). + unwrap_rpc(emqx_proto_v1:get_metrics(Node)). get_stats() -> GlobalStatsKeys = @@ -211,7 +215,7 @@ delete_keys(List, [Key | Keys]) -> delete_keys(proplists:delete(Key, List), Keys). get_stats(Node) -> - wrap_rpc(emqx_proto_v1:get_stats(Node)). + unwrap_rpc(emqx_proto_v1:get_stats(Node)). nodes_info_count(PropList) -> NodeCount = @@ -241,7 +245,7 @@ lookup_client({username, Username}, FormatFun) -> ]). lookup_client(Node, Key, {M, F}) -> - case wrap_rpc(emqx_cm_proto_v1:lookup_client(Node, Key)) of + case unwrap_rpc(emqx_cm_proto_v1:lookup_client(Node, Key)) of {error, Err} -> {error, Err}; L -> @@ -264,7 +268,7 @@ kickout_client({ClientID, FormatFun}) -> end. kickout_client(Node, ClientId) -> - wrap_rpc(emqx_cm_proto_v1:kickout_client(Node, ClientId)). + unwrap_rpc(emqx_cm_proto_v1:kickout_client(Node, ClientId)). list_authz_cache(ClientId) -> call_client(ClientId, list_authz_cache). @@ -284,14 +288,14 @@ list_client_subscriptions(ClientId) -> end. client_subscriptions(Node, ClientId) -> - {Node, wrap_rpc(emqx_broker_proto_v1:list_client_subscriptions(Node, ClientId))}. + {Node, unwrap_rpc(emqx_broker_proto_v1:list_client_subscriptions(Node, ClientId))}. clean_authz_cache(ClientId) -> Results = [clean_authz_cache(Node, ClientId) || Node <- mria_mnesia:running_nodes()], check_results(Results). clean_authz_cache(Node, ClientId) -> - wrap_rpc(emqx_proto_v1:clean_authz_cache(Node, ClientId)). + unwrap_rpc(emqx_proto_v1:clean_authz_cache(Node, ClientId)). clean_authz_cache_all() -> Results = [{Node, clean_authz_cache_all(Node)} || Node <- mria_mnesia:running_nodes()], @@ -308,10 +312,10 @@ wrap_results(Results) -> end. clean_authz_cache_all(Node) -> - wrap_rpc(emqx_proto_v1:clean_authz_cache(Node)). + unwrap_rpc(emqx_proto_v1:clean_authz_cache(Node)). clean_pem_cache_all(Node) -> - wrap_rpc(emqx_proto_v1:clean_pem_cache(Node)). + unwrap_rpc(emqx_proto_v1:clean_pem_cache(Node)). set_ratelimit_policy(ClientId, Policy) -> call_client(ClientId, {ratelimit, Policy}). @@ -357,7 +361,7 @@ do_call_client(ClientId, Req) -> %% @private call_client(Node, ClientId, Req) -> - wrap_rpc(emqx_management_proto_v2:call_client(Node, ClientId, Req)). + unwrap_rpc(emqx_management_proto_v3:call_client(Node, ClientId, Req)). %%-------------------------------------------------------------------- %% Subscriptions @@ -376,7 +380,7 @@ do_list_subscriptions() -> end. list_subscriptions(Node) -> - wrap_rpc(emqx_management_proto_v2:list_subscriptions(Node)). + unwrap_rpc(emqx_management_proto_v3:list_subscriptions(Node)). list_subscriptions_via_topic(Topic, FormatFun) -> lists:append([ @@ -385,7 +389,7 @@ list_subscriptions_via_topic(Topic, FormatFun) -> ]). list_subscriptions_via_topic(Node, Topic, _FormatFun = {M, F}) -> - case wrap_rpc(emqx_broker_proto_v1:list_subscriptions_via_topic(Node, Topic)) of + case unwrap_rpc(emqx_broker_proto_v1:list_subscriptions_via_topic(Node, Topic)) of {error, Reason} -> {error, Reason}; Result -> M:F(Result) end. @@ -394,7 +398,7 @@ lookup_subscriptions(ClientId) -> lists:append([lookup_subscriptions(Node, ClientId) || Node <- mria_mnesia:running_nodes()]). lookup_subscriptions(Node, ClientId) -> - wrap_rpc(emqx_broker_proto_v1:list_client_subscriptions(Node, ClientId)). + unwrap_rpc(emqx_broker_proto_v1:list_client_subscriptions(Node, ClientId)). %%-------------------------------------------------------------------- %% PubSub @@ -404,7 +408,7 @@ subscribe(ClientId, TopicTables) -> subscribe(mria_mnesia:running_nodes(), ClientId, TopicTables). subscribe([Node | Nodes], ClientId, TopicTables) -> - case wrap_rpc(emqx_management_proto_v2:subscribe(Node, ClientId, TopicTables)) of + case unwrap_rpc(emqx_management_proto_v3:subscribe(Node, ClientId, TopicTables)) of {error, _} -> subscribe(Nodes, ClientId, TopicTables); {subscribe, Res} -> {subscribe, Res, Node} end; @@ -431,7 +435,7 @@ unsubscribe(ClientId, Topic) -> -spec unsubscribe([node()], emqx_types:clientid(), emqx_types:topic()) -> {unsubscribe, _} | {error, channel_not_found}. unsubscribe([Node | Nodes], ClientId, Topic) -> - case wrap_rpc(emqx_management_proto_v2:unsubscribe(Node, ClientId, Topic)) of + case unwrap_rpc(emqx_management_proto_v3:unsubscribe(Node, ClientId, Topic)) of {error, _} -> unsubscribe(Nodes, ClientId, Topic); Re -> Re end; @@ -454,7 +458,7 @@ unsubscribe_batch(ClientId, Topics) -> -spec unsubscribe_batch([node()], emqx_types:clientid(), [emqx_types:topic()]) -> {unsubscribe_batch, _} | {error, channel_not_found}. unsubscribe_batch([Node | Nodes], ClientId, Topics) -> - case wrap_rpc(emqx_management_proto_v2:unsubscribe_batch(Node, ClientId, Topics)) of + case unwrap_rpc(emqx_management_proto_v3:unsubscribe_batch(Node, ClientId, Topics)) of {error, _} -> unsubscribe_batch(Nodes, ClientId, Topics); Re -> Re end; @@ -477,16 +481,16 @@ get_alarms(Type) -> [{Node, get_alarms(Node, Type)} || Node <- mria_mnesia:running_nodes()]. get_alarms(Node, Type) -> - add_duration_field(wrap_rpc(emqx_proto_v1:get_alarms(Node, Type))). + add_duration_field(unwrap_rpc(emqx_proto_v1:get_alarms(Node, Type))). deactivate(Node, Name) -> - wrap_rpc(emqx_proto_v1:deactivate_alarm(Node, Name)). + unwrap_rpc(emqx_proto_v1:deactivate_alarm(Node, Name)). delete_all_deactivated_alarms() -> [delete_all_deactivated_alarms(Node) || Node <- mria_mnesia:running_nodes()]. delete_all_deactivated_alarms(Node) -> - wrap_rpc(emqx_proto_v1:delete_all_deactivated_alarms(Node)). + unwrap_rpc(emqx_proto_v1:delete_all_deactivated_alarms(Node)). add_duration_field(Alarms) -> Now = erlang:system_time(microsecond), @@ -523,10 +527,9 @@ delete_banned(Who) -> %%-------------------------------------------------------------------- %% Internal Functions. %%-------------------------------------------------------------------- - -wrap_rpc({badrpc, Reason}) -> +unwrap_rpc({badrpc, Reason}) -> {error, Reason}; -wrap_rpc(Res) -> +unwrap_rpc(Res) -> Res. otp_rel() -> @@ -546,7 +549,7 @@ check_row_limit([Tab | Tables], Limit) -> check_results(Results) -> case lists:any(fun(Item) -> Item =:= ok end, Results) of true -> ok; - false -> wrap_rpc(lists:last(Results)) + false -> unwrap_rpc(lists:last(Results)) end. max_row_limit() -> diff --git a/apps/emqx_management/src/proto/emqx_management_proto_v3.erl b/apps/emqx_management/src/proto/emqx_management_proto_v3.erl new file mode 100644 index 000000000..9c9b71012 --- /dev/null +++ b/apps/emqx_management/src/proto/emqx_management_proto_v3.erl @@ -0,0 +1,80 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_management_proto_v3). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + node_info/1, + broker_info/1, + list_subscriptions/1, + + list_listeners/1, + subscribe/3, + unsubscribe/3, + unsubscribe_batch/3, + + call_client/3, + + get_full_config/1 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.9". + +-spec unsubscribe_batch(node(), emqx_types:clientid(), [emqx_types:topic()]) -> + {unsubscribe, _} | {error, _} | {badrpc, _}. +unsubscribe_batch(Node, ClientId, Topics) -> + rpc:call(Node, emqx_mgmt, do_unsubscribe_batch, [ClientId, Topics]). + +-spec node_info([node()]) -> emqx_rpc:multicall_result(). +node_info(Nodes) -> + erpc:multicall(Nodes, emqx_mgmt, node_info, [], 30000). + +-spec broker_info([node()]) -> emqx_rpc:multicall_result(). +broker_info(Nodes) -> + erpc:multicall(Nodes, emqx_mgmt, broker_info, [], 30000). + +-spec list_subscriptions(node()) -> [map()] | {badrpc, _}. +list_subscriptions(Node) -> + rpc:call(Node, emqx_mgmt, do_list_subscriptions, []). + +-spec list_listeners(node()) -> map() | {badrpc, _}. +list_listeners(Node) -> + rpc:call(Node, emqx_mgmt_api_listeners, do_list_listeners, []). + +-spec subscribe(node(), emqx_types:clientid(), emqx_types:topic_filters()) -> + {subscribe, _} | {error, atom()} | {badrpc, _}. +subscribe(Node, ClientId, TopicTables) -> + rpc:call(Node, emqx_mgmt, do_subscribe, [ClientId, TopicTables]). + +-spec unsubscribe(node(), emqx_types:clientid(), emqx_types:topic()) -> + {unsubscribe, _} | {error, _} | {badrpc, _}. +unsubscribe(Node, ClientId, Topic) -> + rpc:call(Node, emqx_mgmt, do_unsubscribe, [ClientId, Topic]). + +-spec call_client(node(), emqx_types:clientid(), term()) -> term(). +call_client(Node, ClientId, Req) -> + rpc:call(Node, emqx_mgmt, do_call_client, [ClientId, Req]). + +-spec get_full_config(node()) -> map() | list() | {badrpc, _}. +get_full_config(Node) -> + rpc:call(Node, emqx_mgmt_api_configs, get_full_config, []). diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl index 4fb512ed0..197e338fd 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl @@ -113,3 +113,55 @@ t_node_metrics_api(_) -> {error, {_, 400, _}}, emqx_mgmt_api_test_util:request_api(get, BadNodePath) ). + +t_multiple_nodes_api(_) -> + net_kernel:start(['node_api@127.0.0.1', longnames]), + ct:timetrap({seconds, 120}), + snabbkaffe:fix_ct_logging(), + Cluster = [{Name, Opts}, {Name1, Opts1}] = cluster([core, core]), + ct:pal("Starting ~p", [Cluster]), + Node1 = emqx_common_test_helpers:start_slave(Name, Opts), + Node2 = emqx_common_test_helpers:start_slave(Name1, Opts1), + try + NodesPath = emqx_mgmt_api_test_util:api_path(["nodes"]), + {ok, Nodes} = emqx_mgmt_api_test_util:request_api(get, NodesPath), + NodesResponse = emqx_json:decode(Nodes, [return_maps]), + All = [Node1, Node2, node()], + lists:map( + fun(N) -> + N1 = binary_to_atom(maps:get(<<"node">>, N), utf8), + ?assertEqual(true, lists:member(N1, All)) + end, + NodesResponse + ), + ?assertEqual(3, length(NodesResponse)), + + NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(Node1)]), + {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath), + NodeNameResponse = + binary_to_atom(maps:get(<<"node">>, emqx_json:decode(NodeInfo, [return_maps])), utf8), + ?assertEqual(Node1, NodeNameResponse) + after + emqx_common_test_helpers:stop_slave(Node1), + emqx_common_test_helpers:stop_slave(Node2) + end, + ok. + +cluster(Specs) -> + Env = [ + {emqx, init_config_load_done, false}, + {emqx, boot_modules, []} + ], + emqx_common_test_helpers:emqx_cluster(Specs, [ + {env, Env}, + {apps, [emqx_conf]}, + {load_schema, false}, + {join_to, true}, + {env_handler, fun + (emqx) -> + application:set_env(emqx, boot_modules, []), + ok; + (_) -> + ok + end} + ]). From 3066d6662b9d3f546ab1b345de4293f849476c9b Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 25 Oct 2022 07:19:31 +0800 Subject: [PATCH 110/165] chore: add changelog for /node api --- changes/v5.0.10-en.md | 3 +++ changes/v5.0.10-zh.md | 3 +++ 2 files changed, 6 insertions(+) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index 667431852..38da71aad 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -2,6 +2,8 @@ ## Enhancements +- Improve /nodes API speed [#9221](https://github.com/emqx/emqx/pull/9221). + - Allow clear retained/delayed data when client is banned [#9139](https://github.com/emqx/emqx/pull/9139). - Update `gen_rpc` library to version 3.0 [#9187](https://github.com/emqx/emqx/pull/9187). @@ -13,6 +15,7 @@ - Now it is possible to opt out VM internal metrics in prometheus stats [#9222](https://github.com/emqx/emqx/pull/9222). When system load is high, reporting too much metrics data may cause the prometheus stats API timeout. + ## Bug fixes - Fix error log message when `mechanism` is missing in authentication config [#8924](https://github.com/emqx/emqx/pull/8924). diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index b32c21ba6..5c3697d44 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -2,6 +2,8 @@ ## 增强 +- 提升 /nodes API 响应速度 [#9221](https://github.com/emqx/emqx/pull/9221)。 + - 支持拉黑客户端并从数据库中删除保留和延迟发布的消息 [#9139](https://github.com/emqx/emqx/pull/9139)。 - 升级 `gen_rpc` 库到 3.0 [#9187](https://github.com/emqx/emqx/pull/9187)。 @@ -12,6 +14,7 @@ - 可通过配置关闭 prometheus 中的部分内部指标,如果遇到机器负载过高 prometheus 接口返回超时可考虑关闭部分不关心指标,以提高响应速度 [#9222](https://github.com/emqx/emqx/pull/9222)。 + ## Bug fixes - 优化认认证配置中 `mechanism` 字段缺失情况下的错误日志 [#8924](https://github.com/emqx/emqx/pull/8924)。 From af79e652da739d485f75498b1ba4e68b87e38b9a Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 27 Oct 2022 09:39:45 +0800 Subject: [PATCH 111/165] chore: clear dialyzer warning. --- apps/emqx/src/emqx_rpc.erl | 2 +- apps/emqx_management/src/proto/emqx_management_proto_v3.erl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_rpc.erl b/apps/emqx/src/emqx_rpc.erl index ec8fd83de..961bfd5d6 100644 --- a/apps/emqx/src/emqx_rpc.erl +++ b/apps/emqx/src/emqx_rpc.erl @@ -124,7 +124,7 @@ filter_result(Delivery) -> max_client_num() -> emqx:get_config([rpc, tcp_client_num], ?DefaultClientNum). --spec unwrap_erpc(emqx_rpc:erpc(A)) -> A | {error, _Err}. +-spec unwrap_erpc(emqx_rpc:erpc(A) | [emqx_rpc:erpc(A)]) -> A | {error, _Err} | list(). unwrap_erpc(Res) when is_list(Res) -> [unwrap_erpc(R) || R <- Res]; diff --git a/apps/emqx_management/src/proto/emqx_management_proto_v3.erl b/apps/emqx_management/src/proto/emqx_management_proto_v3.erl index 9c9b71012..937a948e5 100644 --- a/apps/emqx_management/src/proto/emqx_management_proto_v3.erl +++ b/apps/emqx_management/src/proto/emqx_management_proto_v3.erl @@ -45,11 +45,11 @@ introduced_in() -> unsubscribe_batch(Node, ClientId, Topics) -> rpc:call(Node, emqx_mgmt, do_unsubscribe_batch, [ClientId, Topics]). --spec node_info([node()]) -> emqx_rpc:multicall_result(). +-spec node_info([node()]) -> emqx_rpc:erpc_multicall(map()). node_info(Nodes) -> erpc:multicall(Nodes, emqx_mgmt, node_info, [], 30000). --spec broker_info([node()]) -> emqx_rpc:multicall_result(). +-spec broker_info([node()]) -> emqx_rpc:erpc_multicall(map()). broker_info(Nodes) -> erpc:multicall(Nodes, emqx_mgmt, broker_info, [], 30000). From 18235a71253b7f58208b52c042f930441f8dcbc5 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 27 Oct 2022 15:03:05 +0800 Subject: [PATCH 112/165] chore: update emqx_managment bpapi.versions --- apps/emqx/priv/bpapi.versions | 1 + .../test/emqx_mgmt_api_nodes_SUITE.erl | 30 ++++++++----------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index da3e5d023..29872f143 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -18,6 +18,7 @@ {emqx_license,2}. {emqx_management,1}. {emqx_management,2}. +{emqx_management,3}. {emqx_mgmt_api_plugins,1}. {emqx_mgmt_cluster,1}. {emqx_mgmt_trace,1}. diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl index 197e338fd..1ba9cdfda 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl @@ -118,29 +118,28 @@ t_multiple_nodes_api(_) -> net_kernel:start(['node_api@127.0.0.1', longnames]), ct:timetrap({seconds, 120}), snabbkaffe:fix_ct_logging(), - Cluster = [{Name, Opts}, {Name1, Opts1}] = cluster([core, core]), + Seq1 = list_to_atom(atom_to_list(?MODULE) ++ "1"), + Seq2 = list_to_atom(atom_to_list(?MODULE) ++ "2"), + Cluster = [{Name, Opts}, {Name1, Opts1}] = cluster([{core, Seq1}, {core, Seq2}]), ct:pal("Starting ~p", [Cluster]), Node1 = emqx_common_test_helpers:start_slave(Name, Opts), Node2 = emqx_common_test_helpers:start_slave(Name1, Opts1), try - NodesPath = emqx_mgmt_api_test_util:api_path(["nodes"]), - {ok, Nodes} = emqx_mgmt_api_test_util:request_api(get, NodesPath), - NodesResponse = emqx_json:decode(Nodes, [return_maps]), - All = [Node1, Node2, node()], + {200, NodesList} = rpc:call(Node1, emqx_mgmt_api_nodes, nodes, [get, #{}]), + All = [Node1, Node2], lists:map( fun(N) -> - N1 = binary_to_atom(maps:get(<<"node">>, N), utf8), + N1 = maps:get(node, N), ?assertEqual(true, lists:member(N1, All)) end, - NodesResponse + NodesList ), - ?assertEqual(3, length(NodesResponse)), + ?assertEqual(2, length(NodesList)), - NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(Node1)]), - {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath), - NodeNameResponse = - binary_to_atom(maps:get(<<"node">>, emqx_json:decode(NodeInfo, [return_maps])), utf8), - ?assertEqual(Node1, NodeNameResponse) + {200, Node11} = rpc:call(Node1, emqx_mgmt_api_nodes, node, [ + get, #{bindings => #{node => Node1}} + ]), + ?assertMatch(#{node := Node1}, Node11) after emqx_common_test_helpers:stop_slave(Node1), emqx_common_test_helpers:stop_slave(Node2) @@ -148,10 +147,7 @@ t_multiple_nodes_api(_) -> ok. cluster(Specs) -> - Env = [ - {emqx, init_config_load_done, false}, - {emqx, boot_modules, []} - ], + Env = [{emqx, boot_modules, []}], emqx_common_test_helpers:emqx_cluster(Specs, [ {env, Env}, {apps, [emqx_conf]}, From 3a491a79847984afe00e8b0e3d517bbc8a9c83cb Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 28 Oct 2022 09:12:56 +0800 Subject: [PATCH 113/165] chore: update changes/v5.0.10-en.md Co-authored-by: Zaiming (Stone) Shi --- changes/v5.0.10-en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index 38da71aad..0b39f7c01 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -2,7 +2,7 @@ ## Enhancements -- Improve /nodes API speed [#9221](https://github.com/emqx/emqx/pull/9221). +- Improve `/nodes` API responsiveness [#9221](https://github.com/emqx/emqx/pull/9221). - Allow clear retained/delayed data when client is banned [#9139](https://github.com/emqx/emqx/pull/9139). From 72163c3056c87c97470f8fa6795063c0ee8b4b5e Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 28 Oct 2022 09:13:07 +0800 Subject: [PATCH 114/165] chore: update changes/v5.0.10-zh.md Co-authored-by: Zaiming (Stone) Shi --- changes/v5.0.10-zh.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 5c3697d44..bbbfb93d8 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -2,7 +2,7 @@ ## 增强 -- 提升 /nodes API 响应速度 [#9221](https://github.com/emqx/emqx/pull/9221)。 +- 提升 `/nodes` API 响应速度 [#9221](https://github.com/emqx/emqx/pull/9221)。 - 支持拉黑客户端并从数据库中删除保留和延迟发布的消息 [#9139](https://github.com/emqx/emqx/pull/9139)。 From 2d548acabd9d783bfc55de84f2c0a4edf43badda Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 28 Oct 2022 08:30:44 +0200 Subject: [PATCH 115/165] style: fix newline --- changes/v5.0.10-en.md | 2 +- changes/v5.0.10-zh.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index 8b25e9418..4bd1ffb6e 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -23,4 +23,4 @@ - Fix HTTP 500 issue when unknown `status` parameter is used in `/gateway` API call [#9225](https://github.com/emqx/emqx/pull/9225). - Fixed the HTTP response status code for the `/status` endpoint [#9211](https://github.com/emqx/emqx/pull/9211). - Before the fix, it always returned `200` even if the EMQX application was not running. Now it returns `503` in that case. \ No newline at end of file + Before the fix, it always returned `200` even if the EMQX application was not running. Now it returns `503` in that case. diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 77e285907..fe9511655 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -22,4 +22,4 @@ - 修复未知 `status` 参数导致 `/gateway` API 发生 HTTP 500 错误的问题 [#9225](https://github.com/emqx/emqx/pull/9225)。 - 修正了 `/status` 端点的响应状态代码 [#9211](https://github.com/emqx/emqx/pull/9211)。 - 在此修复前,它总是返回 HTTP 状态码 `200`,即使 EMQX 没有完成启动或正在重启。 现在它在这些情况下会返回状态码 `503`。 \ No newline at end of file + 在此修复前,它总是返回 HTTP 状态码 `200`,即使 EMQX 没有完成启动或正在重启。 现在它在这些情况下会返回状态码 `503`。 From f920ff341b055b3416648ae94e7c28e1c7e94177 Mon Sep 17 00:00:00 2001 From: Traphalet Date: Thu, 27 Oct 2022 13:28:16 +0300 Subject: [PATCH 116/165] ci(actions): update git actions to latest versions --- .github/actions/package-macos/action.yaml | 2 +- .github/workflows/apps_version_check.yaml | 2 +- .../build_and_push_docker_images.yaml | 24 +++--- .github/workflows/build_packages.yaml | 22 ++--- .github/workflows/build_slim_packages.yaml | 18 ++--- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/code_style_check.yaml | 2 +- .github/workflows/elixir_apps_check.yaml | 2 +- .github/workflows/elixir_deps_check.yaml | 2 +- .github/workflows/elixir_release.yml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/run_emqx_app_tests.yaml | 4 +- .github/workflows/run_fvt_tests.yaml | 10 +-- .github/workflows/run_gitlint.yaml | 2 +- .github/workflows/run_jmeter_tests.yaml | 80 +++++++++++-------- .github/workflows/run_relup_tests.yaml | 10 +-- .github/workflows/run_test_cases.yaml | 20 ++--- .github/workflows/shellcheck.yaml | 2 +- .github/workflows/stale.yaml | 2 +- 19 files changed, 110 insertions(+), 100 deletions(-) diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index 177fdf6b8..57c7910a8 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -33,7 +33,7 @@ runs: brew install curl zip unzip kerl coreutils openssl@1.1 echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH echo "/usr/local/bin" >> $GITHUB_PATH - - uses: actions/cache@v2 + - uses: actions/cache@v3 id: cache with: path: ~/.kerl/${{ inputs.otp }} diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 94d28dd87..13e26b204 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Check apps version diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index ae2cf4fa9..7f659da2f 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -29,7 +29,7 @@ jobs: DOCKER_TAG_VERSION: ${{ steps.get_profile.outputs.DOCKER_TAG_VERSION }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.branch_or_tag }} # when input is not given, the event tag is used path: source @@ -86,7 +86,7 @@ jobs: run: | make -C source deps-all zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: source path: source.zip @@ -146,24 +146,24 @@ jobs: steps: - uses: AutoModality/action-clean@v1 if: matrix.build_machine == 'aws-arm64' - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: source path: . - name: unzip source code run: unzip -q source.zip - - uses: docker/setup-buildx-action@v1 + - uses: docker/setup-buildx-action@v2 - name: Login for docker. - uses: docker/login-action@v1 + uses: docker/login-action@v2 if: matrix.registry == 'docker.io' with: username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - name: Login for AWS ECR - uses: docker/login-action@v1 + uses: docker/login-action@v2 if: matrix.registry == 'public.ecr.aws' with: registry: public.ecr.aws @@ -201,7 +201,7 @@ jobs: echo "::set-output name=img_labels::${img_labels}" # NOTE, Pls make sure this is identical as the one in job 'docker-push-multi-arch-manifest' - - uses: docker/metadata-action@v3 + - uses: docker/metadata-action@v4 id: meta with: images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} @@ -212,7 +212,7 @@ jobs: labels: ${{ steps.pre-meta.outputs.img_labels }} - - uses: docker/build-push-action@v2 + - uses: docker/build-push-action@v3 with: push: ${{ needs.prepare.outputs.IS_EXACT_TAG }} pull: true @@ -275,7 +275,7 @@ jobs: registry: docker.io steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: source path: . @@ -283,13 +283,13 @@ jobs: - name: unzip source code run: unzip -q source.zip - - uses: docker/login-action@v1 + - uses: docker/login-action@v2 if: matrix.registry == 'docker.io' with: username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - - uses: docker/login-action@v1 + - uses: docker/login-action@v2 if: matrix.registry == 'public.ecr.aws' with: registry: public.ecr.aws @@ -327,7 +327,7 @@ jobs: echo "::set-output name=img_labels::${img_labels}" # NOTE, Pls make sure this is identical as the one in job 'docker' - - uses: docker/metadata-action@v3 + - uses: docker/metadata-action@v4 id: meta with: images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 9c7fae257..e4b435a3e 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -29,7 +29,7 @@ jobs: IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.branch_or_tag }} # when input is not given, the event tag is used path: source @@ -79,7 +79,7 @@ jobs: run: | make -C source deps-all zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: source path: source.zip @@ -95,13 +95,13 @@ jobs: otp: - 24.2.1 steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: source path: . - name: unzip source code run: Expand-Archive -Path source.zip -DestinationPath ./ - - uses: ilammy/msvc-dev-cmd@v1 + - uses: ilammy/msvc-dev-cmd@v1.12.0 - uses: erlef/setup-beam@v1 with: otp-version: ${{ matrix.otp }} @@ -127,7 +127,7 @@ jobs: echo "EMQX installed" ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall echo "EMQX uninstalled" - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v3 with: name: ${{ matrix.profile }}-windows path: source/_packages/${{ matrix.profile }}/. @@ -145,7 +145,7 @@ jobs: - macos-11 runs-on: ${{ matrix.os }} steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: source path: . @@ -166,7 +166,7 @@ jobs: apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v3 with: name: ${{ matrix.profile }}-${{ matrix.otp }} path: _packages/${{ matrix.profile }}/. @@ -253,7 +253,7 @@ jobs: steps: - uses: AutoModality/action-clean@v1 if: matrix.build_machine == 'aws-arm64' - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: source path: . @@ -292,7 +292,7 @@ jobs: --elixir "${IsElixir}" \ --builder "ghcr.io/emqx/emqx-builder/5.0-17:${ELIXIR}-${OTP}-${SYSTEM}" done - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v3 with: name: ${{ matrix.profile }}-${{ matrix.otp }} path: source/_packages/${{ matrix.profile }}/. @@ -312,7 +312,7 @@ jobs: - profile: emqx otp: windows # otp version on windows is rather fixed steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: ${{ matrix.profile }}-${{ matrix.otp }} path: packages/${{ matrix.profile }} @@ -334,7 +334,7 @@ jobs: echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1 done cd - - - uses: aws-actions/configure-aws-credentials@v1 + - uses: aws-actions/configure-aws-credentials@v1-node16 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index cf5df532c..8f2e43f45 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -43,7 +43,7 @@ jobs: steps: - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: prepare @@ -73,11 +73,11 @@ jobs: run: | make ${EMQX_NAME}-elixir-pkg ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }} path: _packages/${{ matrix.profile}}/* - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: "${{ matrix.profile }}_schema_dump" path: | @@ -94,8 +94,8 @@ jobs: otp: - 24.2.1 steps: - - uses: actions/checkout@v2 - - uses: ilammy/msvc-dev-cmd@v1 + - uses: actions/checkout@v3 + - uses: ilammy/msvc-dev-cmd@v1.12.0 - uses: erlef/setup-beam@v1 with: otp-version: ${{ matrix.otp }} @@ -119,7 +119,7 @@ jobs: echo "EMQX installed" ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall echo "EMQX uninstalled" - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: windows path: _packages/${{ matrix.profile}}/* @@ -139,7 +139,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: prepare run: | echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV @@ -153,7 +153,7 @@ jobs: apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: macos path: _packages/**/* @@ -167,7 +167,7 @@ jobs: - emqx-enterprise runs-on: aws-amd64 steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 name: Download schema dump with: name: "${{ matrix.profile }}_schema_dump" diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 4a6c31b5e..fdf8903d4 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -8,6 +8,6 @@ jobs: container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run check-deps-integrity.escript run: ./scripts/check-deps-integrity.escript diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml index 5fbf91236..1417960ed 100644 --- a/.github/workflows/code_style_check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1000 - name: Work around https://github.com/actions/checkout/issues/766 diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index 440c91545..2ec54ce63 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -23,7 +23,7 @@ jobs: - name: fix_git_permission run: git config --global --add safe.directory '/__w/emqx/emqx' - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: ensure rebar diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index 312278caa..2cecade04 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: ensure rebar run: ./scripts/ensure-rebar3.sh - name: Work around https://github.com/actions/checkout/issues/766 diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index 006d6aba8..36ba5496d 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: install tools run: apt update && apt install netcat-openbsd - name: Work around https://github.com/actions/checkout/issues/766 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a7abcb244..66b7b04ab 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false steps: - - uses: aws-actions/configure-aws-credentials@v1 + - uses: aws-actions/configure-aws-credentials@v1-node16 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index b2f13e8be..c05fa7a3a 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -31,7 +31,7 @@ jobs: shell: bash steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: run @@ -61,7 +61,7 @@ jobs: ./rebar3 eunit -v ./rebar3 ct -v ./rebar3 proper -d test/props - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v3 if: failure() with: name: logs diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index fbd18f1ce..6a6466442 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -19,7 +19,7 @@ jobs: container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-alpine3.15.1 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: path: source fetch-depth: 0 @@ -27,7 +27,7 @@ jobs: run: | make -C source deps-all zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: source path: source.zip @@ -58,7 +58,7 @@ jobs: - uses: erlef/setup-beam@v1 with: otp-version: "24.2" - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: source path: . @@ -131,7 +131,7 @@ jobs: - uses: erlef/setup-beam@v1 with: otp-version: "24.2" - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: source path: . @@ -207,7 +207,7 @@ jobs: echo "waiting ${{ matrix.profile }} cluster scale" sleep 1 done - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: repository: emqx/paho.mqtt.testing ref: develop-4.0 diff --git a/.github/workflows/run_gitlint.yaml b/.github/workflows/run_gitlint.yaml index 01b35461f..9eb03c0b8 100644 --- a/.github/workflows/run_gitlint.yaml +++ b/.github/workflows/run_gitlint.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout source code - uses: actions/checkout@master + uses: actions/checkout@v3 - name: Install gitlint run: | sudo apt-get update diff --git a/.github/workflows/run_jmeter_tests.yaml b/.github/workflows/run_jmeter_tests.yaml index c2b0442ca..c6d819ba2 100644 --- a/.github/workflows/run_jmeter_tests.yaml +++ b/.github/workflows/run_jmeter_tests.yaml @@ -5,7 +5,7 @@ on: tags: - "v5.*" pull_request: - branchs: + branches: - "master" jobs: @@ -23,11 +23,11 @@ jobs: JMETER_VERSION: 5.4.3 run: | wget --no-verbose --no-check-certificate -O /tmp/apache-jmeter.tgz https://downloads.apache.org/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: apache-jmeter.tgz path: /tmp/apache-jmeter.tgz - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: zip emqx docker image id: build_docker if: endsWith(github.repository, 'emqx') @@ -38,7 +38,7 @@ jobs: VSN="$(./pkg-vsn.sh $PROFILE)" echo "::set-output name=version::${VSN}" docker save -o emqx.tar emqx/emqx:${VSN} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: emqx.tar path: ./emqx.tar @@ -60,8 +60,8 @@ jobs: - uses: erlef/setup-beam@v1 with: otp-version: "24.2" - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: emqx.tar path: /tmp @@ -89,17 +89,19 @@ jobs: done docker ps -a echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: repository: emqx/emqx-fvt ref: broker-autotest path: scripts - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: java-version: '8.0.282' # The JDK version to make available on the path. java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk architecture: x64 # (x64 or x86) - defaults to x64 - - uses: actions/download-artifact@v2 + # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md + distribution: 'zulu' + - uses: actions/download-artifact@v3 with: name: apache-jmeter.tgz path: /tmp @@ -127,7 +129,7 @@ jobs: echo "check logs filed" exit 1 fi - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v3 if: always() with: name: jmeter_logs @@ -154,8 +156,8 @@ jobs: - uses: erlef/setup-beam@v1 with: otp-version: "24.2" - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: emqx.tar path: /tmp @@ -186,17 +188,19 @@ jobs: docker ps -a echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV echo PGSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql-tls) >> $GITHUB_ENV - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: repository: emqx/emqx-fvt ref: broker-autotest path: scripts - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: java-version: '8.0.282' # The JDK version to make available on the path. java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk architecture: x64 # (x64 or x86) - defaults to x64 - - uses: actions/download-artifact@v2 + # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md + distribution: 'zulu' + - uses: actions/download-artifact@v3 with: name: apache-jmeter.tgz path: /tmp @@ -234,7 +238,7 @@ jobs: echo "check logs filed" exit 1 fi - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v3 if: always() with: name: jmeter_logs @@ -258,8 +262,8 @@ jobs: - uses: erlef/setup-beam@v1 with: otp-version: "24.2" - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: emqx.tar path: /tmp @@ -290,17 +294,19 @@ jobs: docker ps -a echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV echo MYSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql-tls) >> $GITHUB_ENV - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: repository: emqx/emqx-fvt ref: broker-autotest path: scripts - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: java-version: '8.0.282' # The JDK version to make available on the path. java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk architecture: x64 # (x64 or x86) - defaults to x64 - - uses: actions/download-artifact@v2 + # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md + distribution: 'zulu' + - uses: actions/download-artifact@v3 with: name: apache-jmeter.tgz path: /tmp @@ -338,7 +344,7 @@ jobs: echo "check logs filed" exit 1 fi - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v3 if: always() with: name: jmeter_logs @@ -358,8 +364,8 @@ jobs: - uses: erlef/setup-beam@v1 with: otp-version: "24.2" - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: emqx.tar path: /tmp @@ -387,7 +393,7 @@ jobs: done docker ps -a echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: repository: emqx/emqx-fvt ref: broker-autotest @@ -400,12 +406,14 @@ jobs: cd target docker run --name jwks_server --network emqx_bridge --ip 172.100.239.88 -d -v $(pwd)/jwkserver-0.0.1.jar:/jwks_server/jwkserver-0.0.1.jar --workdir /jwks_server openjdk:8-jdk bash \ -c "java -jar jwkserver-0.0.1.jar" - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: java-version: '8.0.282' # The JDK version to make available on the path. java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk architecture: x64 # (x64 or x86) - defaults to x64 - - uses: actions/download-artifact@v2 + # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md + distribution: 'zulu' + - uses: actions/download-artifact@v3 with: name: apache-jmeter.tgz path: /tmp @@ -434,7 +442,7 @@ jobs: echo "check logs filed" exit 1 fi - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v3 if: always() with: name: jmeter_logs @@ -455,8 +463,8 @@ jobs: - uses: erlef/setup-beam@v1 with: otp-version: "24.2" - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: emqx.tar path: /tmp @@ -485,17 +493,19 @@ jobs: done docker ps -a echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: repository: emqx/emqx-fvt ref: broker-autotest path: scripts - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: java-version: '8.0.282' # The JDK version to make available on the path. java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk architecture: x64 # (x64 or x86) - defaults to x64 - - uses: actions/download-artifact@v2 + # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md + distribution: 'zulu' + - uses: actions/download-artifact@v3 with: name: apache-jmeter.tgz path: /tmp @@ -524,7 +534,7 @@ jobs: echo "check logs filed" exit 1 fi - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v3 if: always() with: name: jmeter_logs @@ -534,7 +544,7 @@ jobs: runs-on: ubuntu-latest needs: [advanced_feat,pgsql_authn_authz,JWT_authn,mysql_authn_authz,built_in_database_authn_authz] steps: - - uses: geekyeggo/delete-artifact@v1 + - uses: geekyeggo/delete-artifact@v2 with: name: emqx.tar diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 7f33d0a31..6f88c78e4 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -24,7 +24,7 @@ jobs: run: shell: bash steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 name: Checkout with: path: emqx @@ -45,7 +45,7 @@ jobs: cd emqx make emqx-tgz make emqx-enterprise-tgz - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 name: Upload built emqx and test scenario with: name: emqx_built @@ -75,7 +75,7 @@ jobs: - uses: erlef/setup-beam@v1 with: otp-version: "24.2" - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: repository: hawk/lux ref: lux-2.8.1 @@ -88,7 +88,7 @@ jobs: ./configure make echo "$(pwd)/bin" >> $GITHUB_PATH - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 name: Download built emqx and test scenario with: name: emqx_built @@ -114,7 +114,7 @@ jobs: docker logs node2.emqx.io | tee lux_logs/emqx2.log exit 1 fi - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 name: Save debug data if: failure() with: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 540383ed6..53939aaf3 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -22,7 +22,7 @@ jobs: fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }} docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: path: source fetch-depth: 0 @@ -43,7 +43,7 @@ jobs: ./rebar3 as test compile cd .. zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: source path: source.zip @@ -64,7 +64,7 @@ jobs: steps: - uses: AutoModality/action-clean@v1 - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: source path: . @@ -86,7 +86,7 @@ jobs: working-directory: source run: make proper - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: coverdata path: source/_build/test/cover @@ -107,7 +107,7 @@ jobs: steps: - uses: AutoModality/action-clean@v1 - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: source path: . @@ -122,11 +122,11 @@ jobs: REDIS_TAG: 6 run: | ./scripts/ct/run.sh --app ${{ matrix.app_name }} - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v3 with: name: coverdata path: source/_build/test/cover - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v3 if: failure() with: name: logs_${{ matrix.otp_release }}-${{ matrix.profile }} @@ -150,7 +150,7 @@ jobs: steps: - uses: AutoModality/action-clean@v1 - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: source path: . @@ -203,14 +203,14 @@ jobs: container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" steps: - uses: AutoModality/action-clean@v1 - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: source path: . - name: unzip source code run: unzip -q source.zip - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 name: download coverdata with: name: coverdata diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 04a92585a..56a6645e1 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout source code - uses: actions/checkout@master + uses: actions/checkout@v3 - name: Install shellcheck run: | sudo apt-get update diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 378a695cf..32abe1721 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -17,7 +17,7 @@ jobs: steps: - name: Close Stale Issues - uses: actions/stale@v4.1.0 + uses: actions/stale@v6 with: days-before-stale: 7 days-before-close: 7 From ccff64f2c1f4c30ed17fd80da4431930c8aa7b80 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 31 Oct 2022 08:41:22 +0100 Subject: [PATCH 117/165] docs: re-group api tags for gateway apps --- apps/emqx_gateway/src/coap/emqx_coap_api.erl | 3 +- apps/emqx_gateway/src/emqx_gateway_api.erl | 11 +++---- .../src/emqx_gateway_api_authn.erl | 20 +++++++------ .../emqx_gateway_api_authn_user_import.erl | 6 ++-- .../src/emqx_gateway_api_clients.erl | 14 +++++---- .../src/emqx_gateway_api_listeners.erl | 30 ++++++++++--------- .../emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl | 9 +++--- 7 files changed, 52 insertions(+), 41 deletions(-) diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl index 27cb23ac8..f06071040 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl @@ -29,6 +29,7 @@ -export([request/2]). -define(PREFIX, "/gateways/coap/clients/:clientid"). +-define(TAGS, [<<"CoAP Gateways">>]). -import(hoconsc, [mk/2, enum/1]). -import(emqx_dashboard_swagger, [error_codes/2]). @@ -48,7 +49,7 @@ schema(?PREFIX ++ "/request") -> #{ operationId => request, post => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(send_coap_request), summary => <<"Send a Request to a Client">>, parameters => request_parameters(), diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 6ae1135b0..87301e34f 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -54,6 +54,7 @@ ]). -define(KNOWN_GATEWAY_STATUSES, [<<"running">>, <<"stopped">>, <<"unloaded">>]). +-define(TAGS, [<<"Gateways">>]). %%-------------------------------------------------------------------- %% minirest behaviour callbacks @@ -176,7 +177,7 @@ schema("/gateways") -> 'operationId' => gateway, get => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(list_gateway), summary => <<"List All Gateways">>, parameters => params_gateway_status_in_qs(), @@ -192,7 +193,7 @@ schema("/gateways") -> }, post => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(enable_gateway), summary => <<"Enable a Gateway">>, %% TODO: distinguish create & response swagger schema @@ -206,7 +207,7 @@ schema("/gateways/:name") -> 'operationId' => gateway_insta, get => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(get_gateway), summary => <<"Get the Gateway">>, parameters => params_gateway_name_in_path(), @@ -215,7 +216,7 @@ schema("/gateways/:name") -> }, delete => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(delete_gateway), summary => <<"Unload the gateway">>, parameters => params_gateway_name_in_path(), @@ -224,7 +225,7 @@ schema("/gateways/:name") -> }, put => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(update_gateway), summary => <<"Update the gateway confs">>, parameters => params_gateway_name_in_path(), diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl index bb622894b..f337563ee 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl @@ -52,6 +52,8 @@ %% internal export for emqx_gateway_api_listeners module -export([schema_authn/0]). +-define(TAGS, [<<"Gateway Authentication">>]). + %%-------------------------------------------------------------------- %% minirest behaviour callbacks %%-------------------------------------------------------------------- @@ -182,7 +184,7 @@ schema("/gateways/:name/authentication") -> 'operationId' => authn, get => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(get_authn), summary => <<"Get Authenticator Configuration">>, parameters => params_gateway_name_in_path(), @@ -196,7 +198,7 @@ schema("/gateways/:name/authentication") -> }, put => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(update_authn), summary => <<"Update Authenticator Configuration">>, parameters => params_gateway_name_in_path(), @@ -206,7 +208,7 @@ schema("/gateways/:name/authentication") -> }, post => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(add_authn), summary => <<"Create an Authenticator for a Gateway">>, parameters => params_gateway_name_in_path(), @@ -216,7 +218,7 @@ schema("/gateways/:name/authentication") -> }, delete => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(delete_authn), summary => <<"Delete the Gateway Authenticator">>, parameters => params_gateway_name_in_path(), @@ -229,7 +231,7 @@ schema("/gateways/:name/authentication/users") -> 'operationId' => users, get => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(list_users), summary => <<"List users for a Gateway Authenticator">>, parameters => params_gateway_name_in_path() ++ @@ -247,7 +249,7 @@ schema("/gateways/:name/authentication/users") -> }, post => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(add_user), summary => <<"Add User for a Gateway Authenticator">>, parameters => params_gateway_name_in_path(), @@ -271,7 +273,7 @@ schema("/gateways/:name/authentication/users/:uid") -> 'operationId' => users_insta, get => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(get_user), summary => <<"Get User Info for a Gateway Authenticator">>, parameters => params_gateway_name_in_path() ++ @@ -288,7 +290,7 @@ schema("/gateways/:name/authentication/users/:uid") -> }, put => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(update_user), summary => <<"Update User Info for a Gateway Authenticator">>, parameters => params_gateway_name_in_path() ++ @@ -309,7 +311,7 @@ schema("/gateways/:name/authentication/users/:uid") -> }, delete => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(delete_user), summary => <<"Delete User for a Gateway Authenticator">>, parameters => params_gateway_name_in_path() ++ diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl index 0d90b4dc5..b26e77e83 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl @@ -45,6 +45,8 @@ import_listener_users/2 ]). +-define(TAGS, [<<"Gateway Authentication">>]). + %%-------------------------------------------------------------------- %% minirest behaviour callbacks %%-------------------------------------------------------------------- @@ -122,7 +124,7 @@ schema("/gateways/:name/authentication/import_users") -> 'operationId' => import_users, post => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(emqx_gateway_api_authn, import_users), summary => <<"Import Users">>, parameters => params_gateway_name_in_path(), @@ -136,7 +138,7 @@ schema("/gateways/:name/listeners/:id/authentication/import_users") -> 'operationId' => import_listener_users, post => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(emqx_gateway_api_listeners, import_users), summary => <<"Import Users">>, parameters => params_gateway_name_in_path() ++ diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index ab82a4716..2d6275c62 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -59,6 +59,8 @@ format_channel_info/1 ]). +-define(TAGS, [<<"Gateway Clients">>]). + %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -467,7 +469,7 @@ schema("/gateways/:name/clients") -> 'operationId' => clients, get => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(list_clients), summary => <<"List Gateway's Clients">>, parameters => params_client_query(), @@ -480,7 +482,7 @@ schema("/gateways/:name/clients/:clientid") -> 'operationId' => clients_insta, get => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(get_client), summary => <<"Get Client Info">>, parameters => params_client_insta(), @@ -489,7 +491,7 @@ schema("/gateways/:name/clients/:clientid") -> }, delete => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(kick_client), summary => <<"Kick out Client">>, parameters => params_client_insta(), @@ -502,7 +504,7 @@ schema("/gateways/:name/clients/:clientid/subscriptions") -> 'operationId' => subscriptions, get => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(list_subscriptions), summary => <<"List Client's Subscription">>, parameters => params_client_insta(), @@ -518,7 +520,7 @@ schema("/gateways/:name/clients/:clientid/subscriptions") -> }, post => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(add_subscription), summary => <<"Add Subscription for Client">>, parameters => params_client_insta(), @@ -542,7 +544,7 @@ schema("/gateways/:name/clients/:clientid/subscriptions/:topic") -> 'operationId' => subscriptions, delete => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(delete_subscription), summary => <<"Delete Client's Subscription">>, parameters => params_topic_name_in_path() ++ params_client_insta(), diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 259ca8b36..4e4e2be38 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -60,6 +60,8 @@ %% RPC -export([do_listeners_cluster_status/1]). +-define(TAGS, [<<"Gateway Listeners">>]). + %%-------------------------------------------------------------------- %% minirest behaviour callbacks %%-------------------------------------------------------------------- @@ -358,7 +360,7 @@ schema("/gateways/:name/listeners") -> 'operationId' => listeners, get => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(list_listeners), summary => <<"List All Listeners">>, parameters => params_gateway_name_in_path(), @@ -374,7 +376,7 @@ schema("/gateways/:name/listeners") -> }, post => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(add_listener), summary => <<"Add a Listener">>, parameters => params_gateway_name_in_path(), @@ -400,7 +402,7 @@ schema("/gateways/:name/listeners/:id") -> 'operationId' => listeners_insta, get => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(get_listener), summary => <<"Get the Listener Configs">>, parameters => params_gateway_name_in_path() ++ @@ -417,7 +419,7 @@ schema("/gateways/:name/listeners/:id") -> }, delete => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(delete_listener), summary => <<"Delete the Listener">>, parameters => params_gateway_name_in_path() ++ @@ -427,7 +429,7 @@ schema("/gateways/:name/listeners/:id") -> }, put => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(update_listener), summary => <<"Update the Listener Configs">>, parameters => params_gateway_name_in_path() ++ @@ -452,7 +454,7 @@ schema("/gateways/:name/listeners/:id/authentication") -> 'operationId' => listeners_insta_authn, get => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(get_listener_authn), summary => <<"Get the Listener's Authenticator">>, parameters => params_gateway_name_in_path() ++ @@ -467,7 +469,7 @@ schema("/gateways/:name/listeners/:id/authentication") -> }, post => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(add_listener_authn), summary => <<"Create an Authenticator for a Listener">>, parameters => params_gateway_name_in_path() ++ @@ -478,7 +480,7 @@ schema("/gateways/:name/listeners/:id/authentication") -> }, put => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(update_listener_authn), summary => <<"Update the Listener Authenticator configs">>, parameters => params_gateway_name_in_path() ++ @@ -489,7 +491,7 @@ schema("/gateways/:name/listeners/:id/authentication") -> }, delete => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(delete_listener_authn), summary => <<"Delete the Listener's Authenticator">>, parameters => params_gateway_name_in_path() ++ @@ -503,7 +505,7 @@ schema("/gateways/:name/listeners/:id/authentication/users") -> 'operationId' => users, get => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(list_users), summary => <<"List Authenticator's Users">>, parameters => params_gateway_name_in_path() ++ @@ -521,7 +523,7 @@ schema("/gateways/:name/listeners/:id/authentication/users") -> }, post => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(add_user), summary => <<"Add User for an Authenticator">>, parameters => params_gateway_name_in_path() ++ @@ -546,7 +548,7 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") -> 'operationId' => users_insta, get => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(get_user), summary => <<"Get User Info">>, parameters => params_gateway_name_in_path() ++ @@ -564,7 +566,7 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") -> }, put => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(update_user), summary => <<"Update User Info">>, parameters => params_gateway_name_in_path() ++ @@ -586,7 +588,7 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") -> }, delete => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(delete_user), summary => <<"Delete User">>, parameters => params_gateway_name_in_path() ++ diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl index b563d9ea6..9a6468455 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl @@ -27,6 +27,7 @@ -define(PATH(Suffix), "/gateways/lwm2m/clients/:clientid" Suffix). -define(DATA_TYPE, ['Integer', 'Float', 'Time', 'String', 'Boolean', 'Opaque', 'Objlnk']). +-define(TAGS, [<<"LwM2M Gateways">>]). -import(hoconsc, [mk/2, ref/1, ref/2]). -import(emqx_dashboard_swagger, [error_codes/2]). @@ -45,7 +46,7 @@ schema(?PATH("/lookup")) -> #{ 'operationId' => lookup, get => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(lookup_resource), summary => <<"List Client's Resources">>, parameters => [ @@ -70,7 +71,7 @@ schema(?PATH("/observe")) -> #{ 'operationId' => observe, post => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(observe_resource), summary => <<"Observe a Resource">>, parameters => [ @@ -89,7 +90,7 @@ schema(?PATH("/read")) -> #{ 'operationId' => read, post => #{ - tags => [<<"Gateways">>], + tags => ?TAGS, desc => ?DESC(read_resource), summary => <<"Read Value from a Resource Path">>, parameters => [ @@ -106,8 +107,8 @@ schema(?PATH("/write")) -> #{ 'operationId' => write, post => #{ + tags => ?TAGS, desc => ?DESC(write_resource), - tags => [<<"Gateways">>], summary => <<"Write a Value to Resource Path">>, parameters => [ {clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})}, From 935513d38c6272f7ebd1843c98bf85613835cb0a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 31 Oct 2022 14:02:33 -0300 Subject: [PATCH 118/165] docs(fix): remove dynamic example paths from api docs --- apps/emqx_exhook/src/emqx_exhook_api.erl | 6 +++--- apps/emqx_gateway/src/emqx_gateway_api.erl | 4 ++-- .../src/emqx_gateway_api_listeners.erl | 18 +++++++++--------- apps/emqx_gateway/src/emqx_gateway_schema.erl | 7 ++++++- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_api.erl b/apps/emqx_exhook/src/emqx_exhook_api.erl index 921a4df68..6676c4503 100644 --- a/apps/emqx_exhook/src/emqx_exhook_api.erl +++ b/apps/emqx_exhook/src/emqx_exhook_api.erl @@ -219,9 +219,9 @@ params_server_name_in_path() -> server_conf_schema() -> SSL = #{ enable => false, - cacertfile => emqx:cert_file(<<"cacert.pem">>), - certfile => emqx:cert_file(<<"cert.pem">>), - keyfile => emqx:cert_file(<<"key.pem">>) + cacertfile => <<"/etc/emqx/certs/cacert.pem">>, + certfile => <<"/etc/emqx/certs/cert.pem">>, + keyfile => <<"/etc/emqx/certs/key.pem">> }, schema_with_example( ref(server_config), diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 87301e34f..5fbca8b7b 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -686,7 +686,7 @@ examples_gateway_confs() -> enable_stats => true, idle_timeout => <<"30s">>, mountpoint => <<"lwm2m/">>, - xml_dir => emqx:etc_file(<<"lwm2m_xml">>), + xml_dir => <<"/etc/emqx/lwm2m_xml">>, lifetime_min => <<"1s">>, lifetime_max => <<"86400s">>, qmode_time_window => <<"22s">>, @@ -805,7 +805,7 @@ examples_update_gateway_confs() -> enable_stats => true, idle_timeout => <<"30s">>, mountpoint => <<"lwm2m2/">>, - xml_dir => emqx:etc_file(<<"lwm2m_xml">>), + xml_dir => <<"/etc/emqx/lwm2m_xml">>, lifetime_min => <<"1s">>, lifetime_max => <<"86400s">>, qmode_time_window => <<"22s">>, diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 4e4e2be38..08bf37a47 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -756,9 +756,9 @@ examples_listener() -> <<"tlsv1.1">>, <<"tlsv1">> ], - cacertfile => emqx:cert_file(<<"cacert.pem">>), - certfile => emqx:cert_file(<<"cert.pem">>), - keyfile => emqx:cert_file(<<"key.pem">>), + cacertfile => <<"/etc/emqx/certs/cacert.pem">>, + certfile => <<"/etc/emqx/certs/cert.pem">>, + keyfile => <<"/etc/emqx/certs/key.pem">>, verify => <<"verify_none">>, fail_if_no_peer_cert => false }, @@ -802,9 +802,9 @@ examples_listener() -> dtls_options => #{ versions => [<<"dtlsv1.2">>, <<"dtlsv1">>], - cacertfile => emqx:cert_file(<<"cacert.pem">>), - certfile => emqx:cert_file(<<"cert.pem">>), - keyfile => emqx:cert_file(<<"key.pem">>), + cacertfile => <<"/etc/emqx/certs/cacert.pem">>, + certfile => <<"/etc/emqx/certs/cert.pem">>, + keyfile => <<"/etc/emqx/certs/key.pem">>, verify => <<"verify_none">>, fail_if_no_peer_cert => false }, @@ -829,9 +829,9 @@ examples_listener() -> dtls_options => #{ versions => [<<"dtlsv1.2">>, <<"dtlsv1">>], - cacertfile => emqx:cert_file(<<"cacert.pem">>), - certfile => emqx:cert_file(<<"cert.pem">>), - keyfile => emqx:cert_file(<<"key.pem">>), + cacertfile => <<"/etc/emqx/certs/cacert.pem">>, + certfile => <<"/etc/emqx/certs/cert.pem">>, + keyfile => <<"/etc/emqx/certs/key.pem">>, verify => <<"verify_none">>, user_lookup_fun => <<"emqx_tls_psk:lookup">>, ciphers => diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index e5d6dd52a..c58d2f74c 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -250,7 +250,12 @@ fields(lwm2m) -> sc( binary(), #{ - default => emqx:etc_file("lwm2m_xml"), + %% since this is not packaged with emqx, nor + %% present in the packages, we must let the user + %% specify it rather than creating a dynamic + %% default (especially difficult to handle when + %% generating docs). + example => <<"/etc/emqx/lwm2m_xml">>, required => true, desc => ?DESC(lwm2m_xml_dir) } From 198ddda8bc4d83de5776bbf47e28c18a2a4ee3c0 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 31 Oct 2022 14:02:55 -0300 Subject: [PATCH 119/165] docs(status_api): add remark about feature introduction versions --- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 3f844f61d..4e8abef81 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.5"}, + {vsn, "5.0.6"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf index 9c09806cd..fae17b35d 100644 --- a/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf +++ b/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf @@ -8,13 +8,19 @@ emqx_mgmt_api_status { "Returns status code 200 if the EMQX application is up and running, " "503 otherwise." "\n" + "This API was introduced in v5.0.10." + "\n" "The GET `/status` endpoint (without the `/api/...` prefix) is also an alias" - " to this endpoint and works in the same way." + " to this endpoint and works in the same way. This alias has been available since" + " v5.0.0." zh: "作为节点的健康检查。 返回一个纯文本的响应,描述节点的状态。\n" "\n" "如果 EMQX 应用程序已经启动并运行,返回状态代码 200,否则返回 503。\n" "\n" + "这个API是在v5.0.10中引入的。" + "\n" "GET `/status`端点(没有`/api/...`前缀)也是这个端点的一个别名,工作方式相同。" + " 这个别名从v5.0.0开始就有了。" } } From 73b4ac9f656b86f3a173689086f4176a75b6dc2a Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 1 Nov 2022 11:24:24 +0800 Subject: [PATCH 120/165] fix(bridge): keep bridge name type as binary don't convert it to atom After investigation, it was confirmed that there was no need to convert the bridge name to atom --- apps/emqx_bridge/src/emqx_bridge_resource.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index d19cc8426..cbfeda4ba 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -59,9 +59,10 @@ bridge_id(BridgeType, BridgeName) -> Type = bin(BridgeType), <>. +-spec parse_bridge_id(list() | binary() | atom()) -> {atom(), binary()}. parse_bridge_id(BridgeId) -> case string:split(bin(BridgeId), ":", all) of - [Type, Name] -> {binary_to_atom(Type, utf8), binary_to_atom(Name, utf8)}; + [Type, Name] -> {binary_to_atom(Type, utf8), Name}; _ -> error({invalid_bridge_id, BridgeId}) end. From 317e15314ed5c78b074613d5319ae6f302c1bfdd Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 1 Nov 2022 12:38:39 +0800 Subject: [PATCH 121/165] chore: update changes --- changes/v5.0.10-en.md | 1 + changes/v5.0.10-zh.md | 1 + 2 files changed, 2 insertions(+) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index 5b1dd5c77..f9ee82a48 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -15,6 +15,7 @@ - Now it is possible to opt out VM internal metrics in prometheus stats [#9222](https://github.com/emqx/emqx/pull/9222). When system load is high, reporting too much metrics data may cause the prometheus stats API timeout. +- Improve security when converting types such as `binary` `lists` to `atom` types [#9279](https://github.com/emqx/emqx/pull/9279). ## Bug fixes diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 2e058e644..623181497 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -14,6 +14,7 @@ - 可通过配置关闭 prometheus 中的部分内部指标,如果遇到机器负载过高 prometheus 接口返回超时可考虑关闭部分不关心指标,以提高响应速度 [#9222](https://github.com/emqx/emqx/pull/9222)。 +- 提升 `binary` 、`list` 等类型转换为 `atom` 类型时的安全性 [#9279](https://github.com/emqx/emqx/pull/9279)。 ## Bug fixes From 6c52d5de1f4373f58b5c81616a7447404c1f3d65 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 1 Nov 2022 14:41:01 +0800 Subject: [PATCH 122/165] fix(bridge): Ensure that the node name is known --- apps/emqx/src/emqx_misc.erl | 24 +++++++++++++++++- apps/emqx_bridge/src/emqx_bridge_api.erl | 32 ++++++++++++++++-------- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl index 6833cbabb..54157da78 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx/src/emqx_misc.erl @@ -52,7 +52,9 @@ explain_posix/1, pmap/2, pmap/3, - readable_error_msg/1 + readable_error_msg/1, + safe_to_existing_atom/1, + safe_to_existing_atom/2 ]). -export([ @@ -463,6 +465,18 @@ nolink_apply(Fun, Timeout) when is_function(Fun, 0) -> exit(timeout) end. +safe_to_existing_atom(In) -> + safe_to_existing_atom(In, utf8). + +safe_to_existing_atom(Bin, Encoding) when is_binary(Bin) -> + try_to_existing_atom(fun erlang:binary_to_existing_atom/2, [Bin, Encoding]); +safe_to_existing_atom(List, _Encoding) when is_list(List) -> + try_to_existing_atom(fun erlang:list_to_existing_atom/1, [List]); +safe_to_existing_atom(Atom, _Encoding) when is_atom(Atom) -> + {ok, Atom}; +safe_to_existing_atom(_Any, _Encoding) -> + {error, invalid_type}. + %%------------------------------------------------------------------------------ %% Internal Functions %%------------------------------------------------------------------------------ @@ -533,6 +547,14 @@ readable_error_msg(Error) -> end end. +try_to_existing_atom(Fun, Args) -> + try erlang:apply(Fun, Args) of + Atom -> + {ok, Atom} + catch + _:Reason -> {error, Reason} + end. + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 37a42ab3d..66a9079f8 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -492,23 +492,16 @@ lookup_from_local_node(BridgeType, BridgeName) -> invalid -> {400, error_msg('BAD_REQUEST', <<"invalid operation">>)}; OperFunc -> - TargetNode = binary_to_atom(Node, utf8), ConfMap = emqx:get_config([bridges, BridgeType, BridgeName]), case maps:get(enable, ConfMap, false) of false -> {403, error_msg( - 'FORBIDDEN_REQUEST', <<"forbidden operation: bridge disabled">> + 'FORBIDDEN_REQUEST', + <<"forbidden operation: bridge disabled">> )}; true -> - case emqx_bridge_proto_v1:OperFunc(TargetNode, BridgeType, BridgeName) of - ok -> - {200}; - {error, timeout} -> - {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)}; - {error, Reason} -> - {500, error_msg('INTERNAL_ERROR', Reason)} - end + call_operation(Node, OperFunc, BridgeType, BridgeName) end end ). @@ -707,3 +700,22 @@ bin(S) when is_atom(S) -> atom_to_binary(S, utf8); bin(S) when is_binary(S) -> S. + +call_operation(Node, OperFunc, BridgeType, BridgeName) -> + case emqx_misc:safe_to_existing_atom(Node, utf8) of + {ok, TargetNode} -> + case + emqx_bridge_proto_v1:OperFunc( + TargetNode, BridgeType, BridgeName + ) + of + ok -> + {200}; + {error, timeout} -> + {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)}; + {error, Reason} -> + {500, error_msg('INTERNAL_ERROR', Reason)} + end; + {error, _} -> + {400, error_msg('INVALID_NODE', <<"invalid node">>)} + end. From d0687c7aaa24487404e9f6c725ddd2b88042104d Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 1 Nov 2022 15:36:31 +0800 Subject: [PATCH 123/165] fix(connector): keep connector name type as binary don't convert it to atom After investigation, it was confirmed that there was no need to convert the connector name to atom --- apps/emqx_connector/src/emqx_connector.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl index d85959698..1ce4d982a 100644 --- a/apps/emqx_connector/src/emqx_connector.erl +++ b/apps/emqx_connector/src/emqx_connector.erl @@ -78,9 +78,10 @@ connector_id(Type0, Name0) -> Name = bin(Name0), <>. +-spec parse_connector_id(binary() | list() | atom()) -> {atom(), binary()}. parse_connector_id(ConnectorId) -> case string:split(bin(ConnectorId), ":", all) of - [Type, Name] -> {binary_to_atom(Type, utf8), binary_to_atom(Name, utf8)}; + [Type, Name] -> {binary_to_atom(Type, utf8), Name}; _ -> error({invalid_connector_id, ConnectorId}) end. From 60492615ba136c8c7efc9497d4a6c93943911a3f Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 1 Nov 2022 15:49:02 +0800 Subject: [PATCH 124/165] fix(API): Ensure that the node name is known --- .../src/emqx_dashboard_monitor_api.erl | 20 ++++++++++++----- .../src/emqx_gateway_api_clients.erl | 22 +++++++++++-------- .../src/emqx_mgmt_api_clients.erl | 22 +++++++++++-------- .../src/emqx_mgmt_api_subscriptions.erl | 19 ++++++++++------ 4 files changed, 52 insertions(+), 31 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index e3ea870af..50349bc40 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -131,12 +131,20 @@ monitor(get, #{query_string := QS, bindings := Bindings}) -> end. monitor_current(get, #{bindings := Bindings}) -> - NodeOrCluster = binary_to_atom(maps:get(node, Bindings, <<"all">>), utf8), - case emqx_dashboard_monitor:current_rate(NodeOrCluster) of - {ok, CurrentRate} -> - {200, CurrentRate}; - {badrpc, {Node, Reason}} -> - Message = list_to_binary(io_lib:format("Bad node ~p, rpc failed ~p", [Node, Reason])), + RawNode = maps:get(node, Bindings, all), + case emqx_misc:safe_to_existing_atom(RawNode, utf8) of + {ok, NodeOrCluster} -> + case emqx_dashboard_monitor:current_rate(NodeOrCluster) of + {ok, CurrentRate} -> + {200, CurrentRate}; + {badrpc, {Node, Reason}} -> + Message = list_to_binary( + io_lib:format("Bad node ~p, rpc failed ~p", [Node, Reason]) + ), + {400, 'BAD_RPC', Message} + end; + {error, _} -> + Message = list_to_binary(io_lib:format("Bad node ~p", [RawNode])), {400, 'BAD_RPC', Message} end. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index ab82a4716..d5e7791ed 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -113,15 +113,19 @@ clients(get, #{ ?QUERY_FUN ); Node0 -> - Node1 = binary_to_atom(Node0, utf8), - QStringWithoutNode = maps:without([<<"node">>], QString), - emqx_mgmt_api:node_query( - Node1, - QStringWithoutNode, - TabName, - ?CLIENT_QSCHEMA, - ?QUERY_FUN - ) + case emqx_misc:safe_to_existing_atom(Node0) of + {ok, Node1} -> + QStringWithoutNode = maps:without([<<"node">>], QString), + emqx_mgmt_api:node_query( + Node1, + QStringWithoutNode, + TabName, + ?CLIENT_QSCHEMA, + ?QUERY_FUN + ); + {error, _} -> + {error, Node0, {badrpc, <<"invalid node">>}} + end end, case Result of {error, page_limit_invalid} -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 19bf63f66..beff0d53e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -648,15 +648,19 @@ list_clients(QString) -> ?QUERY_FUN ); Node0 -> - Node1 = binary_to_atom(Node0, utf8), - QStringWithoutNode = maps:without([<<"node">>], QString), - emqx_mgmt_api:node_query( - Node1, - QStringWithoutNode, - ?CLIENT_QTAB, - ?CLIENT_QSCHEMA, - ?QUERY_FUN - ) + case emqx_misc:safe_to_existing_atom(Node0) of + {ok, Node1} -> + QStringWithoutNode = maps:without([<<"node">>], QString), + emqx_mgmt_api:node_query( + Node1, + QStringWithoutNode, + ?CLIENT_QTAB, + ?CLIENT_QSCHEMA, + ?QUERY_FUN + ); + {error, _} -> + {error, Node0, {badrpc, <<"invalid node">>}} + end end, case Result of {error, page_limit_invalid} -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 470242cfd..03b833e84 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -145,13 +145,18 @@ subscriptions(get, #{query_string := QString}) -> ?QUERY_FUN ); Node0 -> - emqx_mgmt_api:node_query( - binary_to_atom(Node0, utf8), - QString, - ?SUBS_QTABLE, - ?SUBS_QSCHEMA, - ?QUERY_FUN - ) + case emqx_misc:safe_to_existing_atom(Node0) of + {ok, Node1} -> + emqx_mgmt_api:node_query( + Node1, + QString, + ?SUBS_QTABLE, + ?SUBS_QSCHEMA, + ?QUERY_FUN + ); + {error, _} -> + {error, Node0, {badrpc, <<"invalid node">>}} + end end, case Response of {error, page_limit_invalid} -> From 3cc76c0adfd7f05075a641f7ea9206fac9d23280 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 1 Nov 2022 16:17:08 +0800 Subject: [PATCH 125/165] fix(coap): Ensure that the retain flag is a boolean value --- .../src/coap/handler/emqx_coap_pubsub_handler.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl index 2e962a0bc..9c12e3343 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl +++ b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl @@ -121,7 +121,13 @@ apply_publish_opts(Msg, MQTTMsg) -> maps:fold( fun (<<"retain">>, V, Acc) -> - Val = erlang:binary_to_atom(V), + Val = + case emqx_misc:safe_to_existing_atom(V) of + {ok, true} -> + true; + _ -> + false + end, emqx_message:set_flag(retain, Val, Acc); (<<"expiry">>, V, Acc) -> Val = erlang:binary_to_integer(V), From 612e5acea837e5e5965c456fd38611b17b195aed Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 28 Oct 2022 11:12:10 +0200 Subject: [PATCH 126/165] fix: return 404 instead of 400 for unknown gateway --- apps/emqx_gateway/src/emqx_gateway_api.erl | 2 +- apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl | 11 +++++++---- changes/v5.0.10-en.md | 2 ++ changes/v5.0.10-zh.md | 2 ++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 87301e34f..3f29aaf12 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -151,7 +151,7 @@ gateway_insta(get, #{bindings := #{name := Name0}}) -> end catch error:badarg -> - return_http_error(400, "Bad gateway name") + return_http_error(404, "Bad gateway name") end; gateway_insta(put, #{ body := GwConf0, diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 29c4fefe7..c4a6758a3 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -66,8 +66,8 @@ t_gateway(_) -> lists:foreach(fun assert_gw_unloaded/1, UnloadedGateways), {200, NoRunningGateways} = request(get, "/gateways?status=running"), ?assertEqual([], NoRunningGateways), - {400, BadReqUnknownGw} = request(get, "/gateways/unknown_gateway"), - assert_bad_request(BadReqUnknownGw), + {404, GwNotFoundReq} = request(get, "/gateways/unknown_gateway"), + assert_not_found(GwNotFoundReq), {400, BadReqInvalidStatus} = request(get, "/gateways?status=invalid_status"), assert_bad_request(BadReqInvalidStatus), {400, BadReqUCStatus} = request(get, "/gateways?status=UNLOADED"), @@ -86,8 +86,8 @@ t_gateway(_) -> t_deprecated_gateway(_) -> {200, Gateways} = request(get, "/gateway"), lists:foreach(fun assert_gw_unloaded/1, Gateways), - {400, BadReq} = request(get, "/gateway/uname_gateway"), - assert_bad_request(BadReq), + {404, NotFoundReq} = request(get, "/gateway/uname_gateway"), + assert_not_found(NotFoundReq), {201, _} = request(post, "/gateway", #{name => <<"stomp">>}), {200, StompGw1} = request(get, "/gateway/stomp"), assert_feilds_apperence( @@ -571,3 +571,6 @@ assert_gw_unloaded(Gateway) -> assert_bad_request(BadReq) -> ?assertEqual(<<"BAD_REQUEST">>, maps:get(code, BadReq)). + +assert_not_found(NotFoundReq) -> + ?assertEqual(<<"RESOURCE_NOT_FOUND">>, maps:get(code, NotFoundReq)). diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index 5b1dd5c77..0bd993338 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -29,3 +29,5 @@ This bug was introduced in v5.0.9. For Rule-Engine's input events like `$events/message_delivered` and `$events/message_dropped`, if the message was delivered to a shared-subscription, the encoding (to JSON) of the event will fail. + +- Fix bad HTTP response status code for `/gateways` API, when Gateway name is unknown, it should return `404` instead of `400` [#9268](https://github.com/emqx/emqx/pull/9268). diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 2e058e644..ff818737e 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -27,3 +27,5 @@ - 修复规则引擎的消息事件编码失败 [#9228](https://github.com/emqx/emqx/pull/9228)。 该问题在 v5.0.9 中引入:带消息的规则引擎事件,例如 `$events/message_delivered` 和 `$events/message_dropped`, 如果消息事件是共享订阅产生的,在编码(到 JSON 格式)过程中会失败。 + +- 修复 HTTP API `/gateways` 的返回状态码,未知 Gateway 名字应返回 `404` 而不是 `400` [#9268](https://github.com/emqx/emqx/pull/9268)。 From 881a91a78840aff6c81bfe683870b851d133e94d Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 20 Oct 2022 17:19:18 +0200 Subject: [PATCH 127/165] ci: build elixir images for ce only --- .github/actions/docker-meta/action.yaml | 81 +++++ .../build_and_push_docker_images.yaml | 305 +++++++++--------- scripts/docker-create-push-manifests.sh | 10 +- 3 files changed, 242 insertions(+), 154 deletions(-) create mode 100644 .github/actions/docker-meta/action.yaml diff --git a/.github/actions/docker-meta/action.yaml b/.github/actions/docker-meta/action.yaml new file mode 100644 index 000000000..9ded6d336 --- /dev/null +++ b/.github/actions/docker-meta/action.yaml @@ -0,0 +1,81 @@ +name: 'Docker meta' +inputs: + profile: + required: true + type: string + registry: + required: true + type: string + arch: + required: true + type: string + otp: + required: true + type: string + elixir: + required: false + type: string + default: '' + builder_base: + required: true + type: string + owner: + required: true + type: string + docker_tags: + required: true + type: string + +outputs: + emqx_name: + description: "EMQX name" + value: ${{ steps.pre-meta.outputs.emqx_name }} + version: + description: "docker image version" + value: ${{ steps.meta.outputs.version }} + tags: + description: "docker image tags" + value: ${{ steps.meta.outputs.tags }} + labels: + description: "docker image labels" + value: ${{ steps.meta.outputs.labels }} + +runs: + using: composite + steps: + - name: prepare for docker/metadata-action + id: pre-meta + shell: bash + run: | + emqx_name=${{ inputs.profile }} + img_suffix=${{ inputs.arch }} + img_labels="org.opencontainers.image.otp.version=${{ inputs.otp }}" + if [ -n "${{ inputs.elixir }}" ]; then + emqx_name="emqx-elixir" + img_suffix="elixir-${{ inputs.arch }}" + img_labels="org.opencontainers.image.elixir.version=${{ inputs.elixir }}\n${img_labels}" + fi + if [ ${{ inputs.profile }} = "emqx" ]; then + img_labels="org.opencontainers.image.edition=Opensource\n${img_labels}" + fi + if [ ${{ inputs.profile }} = "emqx-enterprise" ]; then + img_labels="org.opencontainers.image.edition=Enterprise\n${img_labels}" + fi + if [[ ${{ inputs.builder_base }} =~ "alpine" ]]; then + img_suffix="${img_suffix}-alpine" + fi + echo "emqx_name=${emqx_name}" >> $GITHUB_OUTPUT + echo "img_suffix=${img_suffix}" >> $GITHUB_OUTPUT + echo "img_labels=${img_labels}" >> $GITHUB_OUTPUT + echo "img_name=${{ inputs.registry }}/${{ inputs.owner }}/${{ inputs.profile }}" >> $GITHUB_OUTPUT + - uses: docker/metadata-action@v4 + id: meta + with: + images: + ${{ steps.pre-meta.outputs.img_name }} + flavor: | + suffix=-${{ steps.pre-meta.outputs.img_suffix }} + tags: | + type=raw,value=${{ inputs.docker_tags }} + labels: + ${{ steps.pre-meta.outputs.img_labels }} diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 7f659da2f..05b60021e 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -46,7 +46,6 @@ jobs: else docker_latest=false fi - echo "::set-output name=IS_DOCKER_LATEST::${docker_latest}" if git describe --tags --match "[v|e]*" --exact; then echo "This is an exact git tag, will publish images" is_exact='true' @@ -54,7 +53,6 @@ jobs: echo "This is NOT an exact git tag, will not publish images" is_exact='false' fi - echo "::set-output name=IS_EXACT_TAG::${is_exact}" case $tag in refs/tags/v*) PROFILE='emqx' @@ -78,10 +76,12 @@ jobs: esac ;; esac - echo "::set-output name=BUILD_PROFILE::$PROFILE" VSN="$(./pkg-vsn.sh "$PROFILE")" echo "Building $PROFILE image with tag $VSN (latest=$docker_latest)" - echo "::set-output name=DOCKER_TAG_VERSION::$VSN" + echo "IS_DOCKER_LATEST=$docker_latest" >> $GITHUB_OUTPUT + echo "IS_EXACT_TAG=$is_exact" >> $GITHUB_OUTPUT + echo "BUILD_PROFILE=$PROFILE" >> $GITHUB_OUTPUT + echo "DOCKER_TAG_VERSION=$VSN" >> $GITHUB_OUTPUT - name: get_all_deps run: | make -C source deps-all @@ -92,60 +92,33 @@ jobs: path: source.zip docker: - runs-on: ${{ matrix.build_machine }} + runs-on: ${{ matrix.arch[1] }} needs: prepare strategy: fail-fast: false matrix: arch: - - amd64 - - arm64 + - [amd64, ubuntu-20.04] + - [arm64, aws-arm64] profile: - ${{ needs.prepare.outputs.BUILD_PROFILE }} - build_elixir: - - no_elixir registry: - 'docker.io' - 'public.ecr.aws' os: - [alpine3.15.1, "alpine:3.15.1", "deploy/docker/Dockerfile.alpine"] - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] - # NOTE: for docker, only support latest otp and elixir - # versions, not a matrix + # NOTE: 'otp' and 'elixir' are to configure emqx-builder image + # only support latest otp and elixir, not a matrix otp: - 24.2.1-1 # update to latest elixir: - 1.13.4 # update to latest - build_machine: - - aws-arm64 - - ubuntu-20.04 - exclude: - - arch: arm64 - build_machine: ubuntu-20.04 - - arch: amd64 - build_machine: aws-arm64 - include: - - arch: amd64 - profile: emqx - build_elixir: with_elixir - registry: 'docker.io' - os: [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] - otp: 24.2.1-1 - elixir: 1.13.4 - build_machine: ubuntu-20.04 - - arch: arm64 - profile: emqx - build_elixir: with_elixir - registry: 'docker.io' - os: [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] - otp: 24.2.1-1 - elixir: 1.13.4 - build_machine: aws-arm64 steps: - uses: AutoModality/action-clean@v1 - if: matrix.build_machine == 'aws-arm64' + if: matrix.arch[1] == 'aws-arm64' - uses: actions/download-artifact@v3 with: name: source @@ -171,108 +144,123 @@ jobs: password: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ecr: true - - name: prepare for docker-action-parms - id: pre-meta - run: | - emqx_name=${{ matrix.profile }} - img_suffix=${{ matrix.arch }} - img_labels="org.opencontainers.image.otp.version=${{ matrix.otp }}" - - if [ ${{ matrix.build_elixir }} = "with_elixir" ]; then - emqx_name="emqx-elixir" - img_suffix="elixir-${{ matrix.arch }}" - img_labels="org.opencontainers.image.elixir.version=${{ matrix.elixir }}\n${img_labels}" - fi - - if [ ${{ matrix.profile }} = "emqx" ]; then - img_labels="org.opencontainers.image.edition=Opensource\n${img_labels}" - fi - - if [ ${{ matrix.profile }} = "emqx-enterprise" ]; then - img_labels="org.opencontainers.image.edition=Enterprise\n${img_labels}" - fi - - if [[ ${{ matrix.os[0] }} =~ "alpine" ]]; then - img_suffix="${img_suffix}-alpine" - fi - - echo "::set-output name=emqx_name::${emqx_name}" - echo "::set-output name=img_suffix::${img_suffix}" - echo "::set-output name=img_labels::${img_labels}" - - # NOTE, Pls make sure this is identical as the one in job 'docker-push-multi-arch-manifest' - - uses: docker/metadata-action@v4 + - uses: ./source/.github/actions/docker-meta id: meta with: - images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} - flavor: | - suffix=-${{ steps.pre-meta.outputs.img_suffix }} - tags: | - type=raw,value=${{ needs.prepare.outputs.DOCKER_TAG_VERSION }} - labels: - ${{ steps.pre-meta.outputs.img_labels }} + profile: ${{ matrix.profile }} + registry: ${{ matrix.registry }} + arch: ${{ matrix.arch[0] }} + otp: ${{ matrix.otp }} + builder_base: ${{ matrix.os[0] }} + owner: ${{ github.repository_owner }} + docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }} - uses: docker/build-push-action@v3 with: push: ${{ needs.prepare.outputs.IS_EXACT_TAG }} pull: true no-cache: true - platforms: linux/${{ matrix.arch }} + platforms: linux/${{ matrix.arch[0] }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} RUN_FROM=${{ matrix.os[1] }} - EMQX_NAME=${{ steps.pre-meta.outputs.emqx_name }} + EMQX_NAME=${{ steps.meta.outputs.emqx_name }} + file: source/${{ matrix.os[2] }} + context: source + + docker-elixir: + runs-on: ${{ matrix.arch[1] }} + needs: prepare + # do not build elixir images for ee for now + if: needs.prepare.outputs.BUILD_PROFILE == 'emqx' + + strategy: + fail-fast: false + matrix: + arch: + - [amd64, ubuntu-20.04] + - [arm64, aws-arm64] + profile: + - ${{ needs.prepare.outputs.BUILD_PROFILE }} + registry: + - 'docker.io' + os: + - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] + otp: + - 24.2.1-1 # update to latest + elixir: + - 1.13.4 # update to latest + + steps: + - uses: AutoModality/action-clean@v1 + if: matrix.arch[1] == 'aws-arm64' + - uses: actions/download-artifact@v3 + with: + name: source + path: . + - name: unzip source code + run: unzip -q source.zip + + - uses: docker/setup-buildx-action@v2 + + - name: Login for docker. + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - uses: ./source/.github/actions/docker-meta + id: meta + with: + profile: ${{ matrix.profile }} + registry: ${{ matrix.registry }} + arch: ${{ matrix.arch[0] }} + otp: ${{ matrix.otp }} + elixir: ${{ matrix.elixir }} + builder_base: ${{ matrix.os[0] }} + owner: ${{ github.repository_owner }} + docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }} + + - uses: docker/build-push-action@v3 + with: + push: ${{ needs.prepare.outputs.IS_EXACT_TAG }} + pull: true + no-cache: true + platforms: linux/${{ matrix.arch[0] }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + RUN_FROM=${{ matrix.os[1] }} + EMQX_NAME=${{ steps.meta.outputs.emqx_name }} file: source/${{ matrix.os[2] }} context: source docker-push-multi-arch-manifest: # note, we only run on amd64 - if: ${{ needs.prepare.outputs.IS_EXACT_TAG }} + if: needs.prepare.outputs.IS_EXACT_TAG needs: - prepare - docker - runs-on: ubuntu-latest + runs-on: ${{ matrix.arch[1] }} strategy: fail-fast: false matrix: + arch: + - [amd64, ubuntu-20.04] profile: - ${{ needs.prepare.outputs.BUILD_PROFILE }} - build_elixir: - - no_elixir os: - [alpine3.15.1, "alpine:3.15.1", "deploy/docker/Dockerfile.alpine"] - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] - # NOTE: for docker, only support latest otp version, not a matrix + # NOTE: only support latest otp version, not a matrix otp: - 24.2.1-1 # update to latest - # - elixir: - - 1.13.4 # update to latest - arch: - - amd64 - #- arm64 - build_machine: - - aws-arm64 - - ubuntu-20.04 registry: - 'docker.io' - 'public.ecr.aws' - exclude: - - arch: arm64 - build_machine: ubuntu-20.04 - - arch: amd64 - build_machine: aws-arm64 - include: - - arch: amd64 - profile: emqx - build_elixir: with_elixir - os: [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] - otp: 24.2.1-1 - elixir: 1.13.4 - build_machine: ubuntu-20.04 - registry: docker.io steps: - uses: actions/download-artifact@v3 @@ -297,54 +285,73 @@ jobs: password: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ecr: true - - name: prepare for docker-action-parms - id: pre-meta - run: | - emqx_name=${{ matrix.profile }} - img_suffix=${{ matrix.arch }} - img_labels="org.opencontainers.image.otp.version=${{ matrix.otp }}" - - if [ ${{ matrix.build_elixir }} = 'with_elixir' ]; then - emqx_name="emqx-elixir" - img_suffix="elixir-${{ matrix.arch }}" - img_labels="org.opencontainers.image.elixir.version=${{ matrix.elixir }}\n$img_labels" - fi - - if [ ${{ matrix.profile }} = "emqx" ]; then - img_labels="org.opencontainers.image.edition=Opensource\n${img_labels}" - fi - - if [ ${{ matrix.profile }} = "emqx-enterprise" ]; then - img_labels="org.opencontainers.image.edition=Enterprise\n${img_labels}" - fi - - if [[ ${{ matrix.os[0] }} =~ "alpine" ]]; then - img_suffix="${img_suffix}-alpine" - fi - - echo "::set-output name=emqx_name::${emqx_name}" - echo "::set-output name=img_suffix::${img_suffix}" - echo "::set-output name=img_labels::${img_labels}" - - # NOTE, Pls make sure this is identical as the one in job 'docker' - - uses: docker/metadata-action@v4 + - uses: ./source/.github/actions/docker-meta id: meta with: - images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} - flavor: | - suffix=-${{ steps.pre-meta.outputs.img_suffix }} - tags: | - type=raw,value=${{ needs.prepare.outputs.DOCKER_TAG_VERSION }} - labels: - ${{ steps.pre-meta.outputs.img_labels }} + profile: ${{ matrix.profile }} + registry: ${{ matrix.registry }} + arch: ${{ matrix.arch[0] }} + otp: ${{ matrix.otp }} + builder_base: ${{ matrix.os[0] }} + owner: ${{ github.repository_owner }} + docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }} - name: update manifest for multiarch image - if: ${{ needs.prepare.outputs.IS_EXACT_TAG }} working-directory: source run: | - if [ ${{ matrix.build_elixir }} = 'with_elixir' ]; then - is_latest=false - else - is_latest="${{ needs.prepare.outputs.IS_DOCKER_LATEST }}" - fi + is_latest="${{ needs.prepare.outputs.IS_DOCKER_LATEST }}" scripts/docker-create-push-manifests.sh "${{ steps.meta.outputs.tags }}" "$is_latest" + + docker-elixir-push-multi-arch-manifest: + # note, we only run on amd64 + # do not build enterprise elixir images for now + if: needs.prepare.outputs.IS_EXACT_TAG && needs.prepare.outputs.BUILD_PROFILE == 'emqx' + needs: + - prepare + - docker-elixir + runs-on: ${{ matrix.arch[1] }} + strategy: + fail-fast: false + matrix: + arch: + - [amd64, ubuntu-20.04] + profile: + - ${{ needs.prepare.outputs.BUILD_PROFILE }} + # NOTE: for docker, only support latest otp version, not a matrix + otp: + - 24.2.1-1 # update to latest + elixir: + - 1.13.4 # update to latest + registry: + - 'docker.io' + + steps: + - uses: actions/download-artifact@v3 + with: + name: source + path: . + + - name: unzip source code + run: unzip -q source.zip + + - uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - uses: ./source/.github/actions/docker-meta + id: meta + with: + profile: ${{ matrix.profile }} + registry: ${{ matrix.registry }} + arch: ${{ matrix.arch[0] }} + otp: ${{ matrix.otp }} + elixir: ${{ matrix.elixir }} + builder_base: ${{ matrix.os[0] }} + owner: ${{ github.repository_owner }} + docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }} + + - name: update manifest for multiarch image + working-directory: source + run: | + scripts/docker-create-push-manifests.sh "${{ steps.meta.outputs.tags }}" false diff --git a/scripts/docker-create-push-manifests.sh b/scripts/docker-create-push-manifests.sh index 7c67ae788..db9c01bfb 100755 --- a/scripts/docker-create-push-manifests.sh +++ b/scripts/docker-create-push-manifests.sh @@ -2,23 +2,23 @@ set -exuo pipefail img_amd64=$1 -IsPushLatest=$2 +push_latest=${2:-false} img_arm64=$(echo ${img_amd64} | sed 's/-amd64$/-arm64/g') -img_march=${img_amd64%-amd64} +img_name=${img_amd64%-amd64} docker pull "$img_amd64" docker pull --platform linux/arm64 "$img_arm64" img_amd64_digest=$(docker inspect --format='{{index .RepoDigests 0}}' "$img_amd64") img_arm64_digest=$(docker inspect --format='{{index .RepoDigests 0}}' "$img_arm64") echo "sha256 of amd64 is $img_amd64_digest" echo "sha256 of arm64 is $img_arm64_digest" -docker manifest create "${img_march}" \ +docker manifest create "${img_name}" \ --amend "$img_amd64_digest" \ --amend "$img_arm64_digest" -docker manifest push "${img_march}" +docker manifest push "${img_name}" # PUSH latest if it is a release build -if [ "$IsPushLatest" = "true" ]; then +if [ "$push_latest" = "true" ]; then img_latest=$(echo "$img_arm64" | cut -d: -f 1):latest docker manifest create "${img_latest}" \ --amend "$img_amd64_digest" \ From e097ab628e793be0a60bb7cbaad7d068b1b3ce4d Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 1 Nov 2022 16:18:08 +0100 Subject: [PATCH 128/165] build: add quotes around variables in action shell script --- .github/actions/docker-meta/action.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/docker-meta/action.yaml b/.github/actions/docker-meta/action.yaml index 9ded6d336..13ab21da6 100644 --- a/.github/actions/docker-meta/action.yaml +++ b/.github/actions/docker-meta/action.yaml @@ -55,13 +55,13 @@ runs: img_suffix="elixir-${{ inputs.arch }}" img_labels="org.opencontainers.image.elixir.version=${{ inputs.elixir }}\n${img_labels}" fi - if [ ${{ inputs.profile }} = "emqx" ]; then + if [ "${{ inputs.profile }}" = "emqx" ]; then img_labels="org.opencontainers.image.edition=Opensource\n${img_labels}" fi - if [ ${{ inputs.profile }} = "emqx-enterprise" ]; then + if [ "${{ inputs.profile }}" = "emqx-enterprise" ]; then img_labels="org.opencontainers.image.edition=Enterprise\n${img_labels}" fi - if [[ ${{ inputs.builder_base }} =~ "alpine" ]]; then + if [[ "${{ inputs.builder_base }}" =~ "alpine" ]]; then img_suffix="${img_suffix}-alpine" fi echo "emqx_name=${emqx_name}" >> $GITHUB_OUTPUT From 7ff7fb5a2399ee224480253f43ee82166aaf6501 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 1 Nov 2022 16:23:26 +0100 Subject: [PATCH 129/165] ci: bump login action version --- .github/workflows/build_and_push_docker_images.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 05b60021e..803b50cfc 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -206,7 +206,7 @@ jobs: - uses: docker/setup-buildx-action@v2 - name: Login for docker. - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_TOKEN }} @@ -334,7 +334,7 @@ jobs: - name: unzip source code run: unzip -q source.zip - - uses: docker/login-action@v1 + - uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_TOKEN }} From bb7476d3af104330e3a0191c726616684475f895 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 2 Nov 2022 14:45:47 +0800 Subject: [PATCH 130/165] fix(listeners): Constraints the atom convert when parsing the esockd access rules esockd rules only use words 'allow' and 'deny', both are existing, so it is better to restrict the conversion and print a log when errors --- apps/emqx/src/emqx_listeners.erl | 32 +++++++++++++------ apps/emqx/src/emqx_misc.erl | 10 +++--- .../coap/handler/emqx_coap_pubsub_handler.erl | 8 +---- apps/emqx_gateway/src/emqx_gateway_utils.erl | 15 ++------- 4 files changed, 30 insertions(+), 35 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 2a809f88d..a0f2b1e7d 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -49,7 +49,8 @@ -export([ listener_id/2, parse_listener_id/1, - ensure_override_limiter_conf/2 + ensure_override_limiter_conf/2, + esockd_access_rules/1 ]). -export([pre_config_update/3, post_config_update/5]). @@ -497,17 +498,28 @@ ip_port({Addr, Port}) -> [{ip, Addr}, {port, Port}]. esockd_access_rules(StrRules) -> - Access = fun(S) -> + Access = fun(S, Acc) -> [A, CIDR] = string:tokens(S, " "), - { - list_to_atom(A), - case CIDR of - "all" -> all; - _ -> CIDR - end - } + %% esockd rules only use words 'allow' and 'deny', both are existing + %% comparison of strings may be better, but there is a loss of backward compatibility + case emqx_misc:safe_to_existing_atom(A) of + {ok, Action} -> + [ + { + Action, + case CIDR of + "all" -> all; + _ -> CIDR + end + } + | Acc + ]; + _ -> + ?SLOG(warning, #{msg => "invalid esockd access rule", rule => S}), + Acc + end end, - [Access(R) || R <- StrRules]. + lists:foldr(Access, [], StrRules). merge_default(Options) -> case lists:keytake(tcp_options, 1, Options) of diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl index 54157da78..674465e6a 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx/src/emqx_misc.erl @@ -469,9 +469,9 @@ safe_to_existing_atom(In) -> safe_to_existing_atom(In, utf8). safe_to_existing_atom(Bin, Encoding) when is_binary(Bin) -> - try_to_existing_atom(fun erlang:binary_to_existing_atom/2, [Bin, Encoding]); -safe_to_existing_atom(List, _Encoding) when is_list(List) -> - try_to_existing_atom(fun erlang:list_to_existing_atom/1, [List]); + try_to_existing_atom(fun erlang:binary_to_existing_atom/2, Bin, Encoding); +safe_to_existing_atom(List, Encoding) when is_list(List) -> + try_to_existing_atom(fun(In, _) -> erlang:list_to_existing_atom(In) end, List, Encoding); safe_to_existing_atom(Atom, _Encoding) when is_atom(Atom) -> {ok, Atom}; safe_to_existing_atom(_Any, _Encoding) -> @@ -547,8 +547,8 @@ readable_error_msg(Error) -> end end. -try_to_existing_atom(Fun, Args) -> - try erlang:apply(Fun, Args) of +try_to_existing_atom(Convert, Data, Encoding) -> + try Convert(Data, Encoding) of Atom -> {ok, Atom} catch diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl index 9c12e3343..8587dc1dc 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl +++ b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl @@ -121,13 +121,7 @@ apply_publish_opts(Msg, MQTTMsg) -> maps:fold( fun (<<"retain">>, V, Acc) -> - Val = - case emqx_misc:safe_to_existing_atom(V) of - {ok, true} -> - true; - _ -> - false - end, + Val = V =:= <<"true">>, emqx_message:set_flag(retain, Val, Acc); (<<"expiry">>, V, Acc) -> Val = erlang:binary_to_integer(V), diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 8df7d84c0..68fce7589 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -70,6 +70,8 @@ default_subopts/0 ]). +-import(emqx_listeners, [esockd_access_rules/1]). + -define(ACTIVE_N, 100). -define(DEFAULT_IDLE_TIMEOUT, 30000). -define(DEFAULT_GC_OPTS, #{count => 1000, bytes => 1024 * 1024}). @@ -443,19 +445,6 @@ esockd_opts(Type, Opts0) -> end ). -esockd_access_rules(StrRules) -> - Access = fun(S) -> - [A, CIDR] = string:tokens(S, " "), - { - list_to_atom(A), - case CIDR of - "all" -> all; - _ -> CIDR - end - } - end, - [Access(R) || R <- StrRules]. - ssl_opts(Name, Opts) -> Type = case Name of From 14c8b5ebb15294a1ff2366933dd7b25b3a14b17a Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 2 Nov 2022 16:05:30 +0800 Subject: [PATCH 131/165] fix(mgmt): Ensure the CLI will not create useless atom The log level, the log handler id and the listener id all of them already exist, so it's better to avoid the user create a useless atom by mistake input --- apps/emqx_management/src/emqx_mgmt_cli.erl | 120 +++++++++++++++------ 1 file changed, 85 insertions(+), 35 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 60b2f3b15..6a0f8e8be 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -356,15 +356,26 @@ mnesia(_) -> %% @doc Logger Command log(["set-level", Level]) -> - case emqx_logger:set_log_level(list_to_atom(Level)) of - ok -> emqx_ctl:print("~ts~n", [Level]); - Error -> emqx_ctl:print("[error] set overall log level failed: ~p~n", [Error]) + case emqx_misc:safe_to_existing_atom(Level) of + {ok, Level1} -> + case emqx_logger:set_log_level(Level1) of + ok -> emqx_ctl:print("~ts~n", [Level]); + Error -> emqx_ctl:print("[error] set overall log level failed: ~p~n", [Error]) + end; + _ -> + emqx_ctl:print("[error] invalid level: ~p~n", [Level]) end; log(["primary-level"]) -> Level = emqx_logger:get_primary_log_level(), emqx_ctl:print("~ts~n", [Level]); log(["primary-level", Level]) -> - _ = emqx_logger:set_primary_log_level(list_to_atom(Level)), + case emqx_misc:safe_to_existing_atom(Level) of + {ok, Level1} -> + _ = emqx_logger:set_primary_log_level(Level1), + ok; + _ -> + emqx_ctl:print("[error] invalid level: ~p~n", [Level]) + end, emqx_ctl:print("~ts~n", [emqx_logger:get_primary_log_level()]); log(["handlers", "list"]) -> _ = [ @@ -381,26 +392,50 @@ log(["handlers", "list"]) -> ], ok; log(["handlers", "start", HandlerId]) -> - case emqx_logger:start_log_handler(list_to_atom(HandlerId)) of - ok -> - emqx_ctl:print("log handler ~ts started~n", [HandlerId]); - {error, Reason} -> - emqx_ctl:print("[error] failed to start log handler ~ts: ~p~n", [HandlerId, Reason]) + case emqx_misc:safe_to_existing_atom(HandlerId) of + {ok, HandlerId1} -> + case emqx_logger:start_log_handler(HandlerId1) of + ok -> + emqx_ctl:print("log handler ~ts started~n", [HandlerId]); + {error, Reason} -> + emqx_ctl:print("[error] failed to start log handler ~ts: ~p~n", [ + HandlerId, Reason + ]) + end; + _ -> + emqx_ctl:print("[error] invalid handler:~ts~n", [HandlerId]) end; log(["handlers", "stop", HandlerId]) -> - case emqx_logger:stop_log_handler(list_to_atom(HandlerId)) of - ok -> - emqx_ctl:print("log handler ~ts stopped~n", [HandlerId]); - {error, Reason} -> - emqx_ctl:print("[error] failed to stop log handler ~ts: ~p~n", [HandlerId, Reason]) + case emqx_misc:safe_to_existing_atom(HandlerId) of + {ok, HandlerId1} -> + case emqx_logger:stop_log_handler(HandlerId1) of + ok -> + emqx_ctl:print("log handler ~ts stopped~n", [HandlerId1]); + {error, Reason} -> + emqx_ctl:print("[error] failed to stop log handler ~ts: ~p~n", [ + HandlerId1, Reason + ]) + end; + _ -> + emqx_ctl:print("[error] invalid handler:~ts~n", [HandlerId]) end; log(["handlers", "set-level", HandlerId, Level]) -> - case emqx_logger:set_log_handler_level(list_to_atom(HandlerId), list_to_atom(Level)) of - ok -> - #{level := NewLevel} = emqx_logger:get_log_handler(list_to_atom(HandlerId)), - emqx_ctl:print("~ts~n", [NewLevel]); - {error, Error} -> - emqx_ctl:print("[error] ~p~n", [Error]) + case emqx_misc:safe_to_existing_atom(HandlerId) of + {ok, HandlerId1} -> + case emqx_misc:safe_to_existing_atom(Level) of + {ok, Level1} -> + case emqx_logger:set_log_handler_level(HandlerId1, Level1) of + ok -> + #{level := NewLevel} = emqx_logger:get_log_handler(HandlerId1), + emqx_ctl:print("~ts~n", [NewLevel]); + {error, Error} -> + emqx_ctl:print("[error] ~p~n", [Error]) + end; + _ -> + emqx_ctl:print("[error] invalid level:~p~n", [Level]) + end; + _ -> + emqx_ctl:print("[error] invalid handler:~ts~n", [HandlerId]) end; log(_) -> emqx_ctl:usage( @@ -593,25 +628,40 @@ listeners([]) -> emqx_listeners:list() ); listeners(["stop", ListenerId]) -> - case emqx_listeners:stop_listener(list_to_atom(ListenerId)) of - ok -> - emqx_ctl:print("Stop ~ts listener successfully.~n", [ListenerId]); - {error, Error} -> - emqx_ctl:print("Failed to stop ~ts listener: ~0p~n", [ListenerId, Error]) + case emqx_misc:safe_to_existing_atom(ListenerId) of + {ok, ListenerId1} -> + case emqx_listeners:stop_listener(ListenerId1) of + ok -> + emqx_ctl:print("Stop ~ts listener successfully.~n", [ListenerId]); + {error, Error} -> + emqx_ctl:print("Failed to stop ~ts listener: ~0p~n", [ListenerId, Error]) + end; + _ -> + emqx_ctl:print("Invalid listener: ~0p~n", [ListenerId]) end; listeners(["start", ListenerId]) -> - case emqx_listeners:start_listener(list_to_atom(ListenerId)) of - ok -> - emqx_ctl:print("Started ~ts listener successfully.~n", [ListenerId]); - {error, Error} -> - emqx_ctl:print("Failed to start ~ts listener: ~0p~n", [ListenerId, Error]) + case emqx_misc:safe_to_existing_atom(ListenerId) of + {ok, ListenerId1} -> + case emqx_listeners:start_listener(ListenerId1) of + ok -> + emqx_ctl:print("Started ~ts listener successfully.~n", [ListenerId]); + {error, Error} -> + emqx_ctl:print("Failed to start ~ts listener: ~0p~n", [ListenerId, Error]) + end; + _ -> + emqx_ctl:print("Invalid listener: ~0p~n", [ListenerId]) end; listeners(["restart", ListenerId]) -> - case emqx_listeners:restart_listener(list_to_atom(ListenerId)) of - ok -> - emqx_ctl:print("Restarted ~ts listener successfully.~n", [ListenerId]); - {error, Error} -> - emqx_ctl:print("Failed to restart ~ts listener: ~0p~n", [ListenerId, Error]) + case emqx_misc:safe_to_existing_atom(ListenerId) of + {ok, ListenerId1} -> + case emqx_listeners:restart_listener(ListenerId1) of + ok -> + emqx_ctl:print("Restarted ~ts listener successfully.~n", [ListenerId]); + {error, Error} -> + emqx_ctl:print("Failed to restart ~ts listener: ~0p~n", [ListenerId, Error]) + end; + _ -> + emqx_ctl:print("Invalid listener: ~0p~n", [ListenerId]) end; listeners(_) -> emqx_ctl:usage([ From 7401cec7120c509d74d87b18e48ad32bdb3e2e45 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 3 Nov 2022 13:04:59 -0700 Subject: [PATCH 132/165] ci: fix helm chart upload on releases emqx/push-helm-action requires access to charts in the source tree, but we do not run checkout in the main job --- .github/workflows/release.yaml | 46 +++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 66b7b04ab..d1fb8ef9e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -57,24 +57,6 @@ jobs: -X POST \ -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - - uses: emqx/push-helm-action@v1 - if: github.event_name == 'release' && startsWith(github.ref_name, 'v') - with: - charts_dir: "${{ github.workspace }}/deploy/charts/emqx" - version: ${{ github.ref_name }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-west-2" - aws_bucket_name: "repos-emqx-io" - - uses: emqx/push-helm-action@v1 - if: github.event_name == 'release' && startsWith(github.ref_name, 'e') - with: - charts_dir: "${{ github.workspace }}/deploy/charts/emqx-enterprise" - version: ${{ github.ref_name }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-west-2" - aws_bucket_name: "repos-emqx-io" - name: update homebrew packages if: github.event_name == 'release' run: | @@ -96,3 +78,31 @@ jobs: -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ github.ref_name }}\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches" fi + + upload-helm: + runs-on: ubuntu-20.04 + if: github.event_name == 'release' + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.ref } + - uses: emqx/push-helm-action@v1 + if: startsWith(github.ref_name, 'v') + with: + charts_dir: "${{ github.workspace }}/deploy/charts/emqx" + version: ${{ github.ref_name }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: "us-west-2" + aws_bucket_name: "repos-emqx-io" + - uses: emqx/push-helm-action@v1 + if: startsWith(github.ref_name, 'e') + with: + charts_dir: "${{ github.workspace }}/deploy/charts/emqx-enterprise" + version: ${{ github.ref_name }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: "us-west-2" + aws_bucket_name: "repos-emqx-io" From bd13ae0ad6f529dfb1f53ee9e15604db0edc0643 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 3 Nov 2022 14:18:28 +0800 Subject: [PATCH 133/165] fix(acl): Check the real topic in delayed messages We need to check the true topic of the delayed message correctly the cheapest way to do this is to extract the true topic from the original topic when doing ACL check --- apps/emqx/src/emqx_access_control.erl | 18 +++++++++++++ apps/emqx/test/emqx_access_control_SUITE.erl | 28 ++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index 1345c78e0..66d45b29a 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -24,6 +24,11 @@ authorize/3 ]). +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -45,6 +50,19 @@ authenticate(Credential) -> %% @doc Check Authorization -spec authorize(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic()) -> allow | deny. +authorize(ClientInfo, PubSub, <<"$delayed/", Data/binary>> = RawTopic) -> + case binary:split(Data, <<"/">>) of + [_, Topic] -> + authorize(ClientInfo, PubSub, Topic); + _ -> + ?SLOG(warning, #{ + msg => "invalid_dealyed_topic_format", + expected_example => "$delayed/1/t/foo", + got => RawTopic + }), + inc_authz_metrics(deny), + deny + end; authorize(ClientInfo, PubSub, Topic) -> Result = case emqx_authz_cache:is_enabled() of diff --git a/apps/emqx/test/emqx_access_control_SUITE.erl b/apps/emqx/test/emqx_access_control_SUITE.erl index 23c43fa65..ee594ec0a 100644 --- a/apps/emqx/test/emqx_access_control_SUITE.erl +++ b/apps/emqx/test/emqx_access_control_SUITE.erl @@ -32,6 +32,12 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([]). +end_per_testcase(t_delayed_authorize, Config) -> + meck:unload(emqx_access_control), + Config; +end_per_testcase(_, Config) -> + Config. + t_authenticate(_) -> ?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())). @@ -39,6 +45,28 @@ t_authorize(_) -> Publish = ?PUBLISH_PACKET(?QOS_0, <<"t">>, 1, <<"payload">>), ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish, <<"t">>)). +t_delayed_authorize(_) -> + RawTopic = "$dealyed/1/foo/2", + InvalidTopic = "$dealyed/1/foo/3", + Topic = "foo/2", + + ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), + ok = meck:expect( + emqx_access_control, + do_authorize, + fun + (_, _, Topic) -> allow; + (_, _, _) -> deny + end + ), + + Publish1 = ?PUBLISH_PACKET(?QOS_0, RawTopic, 1, <<"payload">>), + ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish1, RawTopic)), + + Publish2 = ?PUBLISH_PACKET(?QOS_0, InvalidTopic, 1, <<"payload">>), + ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish2, InvalidTopic)), + ok. + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- From 1d0b30ab5e26f61bd96b848dec5f0dd25808406d Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 4 Nov 2022 10:42:08 +0800 Subject: [PATCH 134/165] chore: update changes --- changes/v5.0.10-en.md | 3 +++ changes/v5.0.10-zh.md | 3 +++ 2 files changed, 6 insertions(+) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index fe3aa46e6..9ec10d3a7 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -32,3 +32,6 @@ the encoding (to JSON) of the event will fail. - Fix bad HTTP response status code for `/gateways` API, when Gateway name is unknown, it should return `404` instead of `400` [#9268](https://github.com/emqx/emqx/pull/9268). + +- Fix incorrect topic authorize checking of delayed messages [#9290](https://github.com/emqx/emqx/pull/9290). + Now will determine the actual topic of the delayed messages, e.g. `$delayed/1/t/foo` will be treated as `t/foo` in authorize checks. diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 01c49b49e..d7e758162 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -30,3 +30,6 @@ `$events/message_dropped`, 如果消息事件是共享订阅产生的,在编码(到 JSON 格式)过程中会失败。 - 修复 HTTP API `/gateways` 的返回状态码,未知 Gateway 名字应返回 `404` 而不是 `400` [#9268](https://github.com/emqx/emqx/pull/9268)。 + +- 修复延迟消息的主题授权判断不正确的问题 [#9290](https://github.com/emqx/emqx/pull/9290)。 + 现在将会对延迟消息中的真实主题进行授权判断,比如,`$delayed/1/t/foo` 会被当作 `t/foo` 进行判断。 From 1d18c51d4a325119d44e734f6feb58e4bd132512 Mon Sep 17 00:00:00 2001 From: Traphalet Date: Fri, 4 Nov 2022 10:27:15 +0200 Subject: [PATCH 135/165] chore(actions): update version for leftover action --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 66b7b04ab..42922a76d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -41,7 +41,7 @@ jobs: echo "$fname -> $default_fname" mv -f "$fname" "$default_fname" done < <(find . -maxdepth 1 -type f | grep -E "emqx(-enterprise)?-5\.[0-9]+\.[0-9]+.*-${DEFAULT_BEAM_PLATFORM}" | grep -v elixir) - - uses: alexellis/upload-assets@0.2.2 + - uses: alexellis/upload-assets@0.4.0 env: GITHUB_TOKEN: ${{ github.token }} with: From 4c4fbcaab07f82c654a44e1b0543a8304bfcd721 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 4 Nov 2022 13:34:24 +0100 Subject: [PATCH 136/165] chore: update pr template --- .github/PULL_REQUEST_TEMPLATE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 41c6276a3..38c008b04 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -10,7 +10,8 @@ Please convert it to a draft if any of the following conditions are not met. Rev - [ ] Changed lines covered in coverage report - [ ] Change log has been added to `changes/` dir - [ ] For EMQX 4.x: `appup` files updated (execute `scripts/update-appup.sh emqx`) -- [ ] For internal contributor: there is a jira ticket to track this change, and another jira tickt to track doc updates (if any) +- [ ] For internal contributor: there is a jira ticket to track this change +- [ ] If there should be document changes, a PR to emqx-docs.git is sent, or a jira ticket is created to follow up - [ ] In case of non-backward compatible changes, reviewer should check this item as a write-off, and add details in **Backward Compatibility** section ## Backward Compatibility From 8b10b78bce917e46d420944c813de70695a02cb0 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 3 Nov 2022 17:04:11 +0100 Subject: [PATCH 137/165] fix(emqx_mgmt_api_configs): use 'node' query paramter if given also fix result in error case - be compliant to schema and also return correct http status code --- apps/emqx_management/src/emqx_mgmt_api_configs.erl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index e38b6b729..db582c612 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -103,7 +103,9 @@ schema("/configs") -> )} ], responses => #{ - 200 => lists:map(fun({_, Schema}) -> Schema end, config_list()) + 200 => lists:map(fun({_, Schema}) -> Schema end, config_list()), + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND']), + 500 => emqx_dashboard_swagger:error_codes(['BAD_NODE']) } } }; @@ -311,14 +313,15 @@ config_reset(post, _Params, Req) -> end. configs(get, Params, _Req) -> - Node = maps:get(node, Params, node()), + QS = maps:get(query_string, Params, #{}), + Node = maps:get(<<"node">>, QS, node()), case lists:member(Node, mria_mnesia:running_nodes()) andalso emqx_management_proto_v2:get_full_config(Node) of false -> Message = list_to_binary(io_lib:format("Bad node ~p, reason not found", [Node])), - {500, #{code => 'BAD_NODE', message => Message}}; + {404, #{code => 'NOT_FOUND', message => Message}}; {badrpc, R} -> Message = list_to_binary(io_lib:format("Bad node ~p, reason ~p", [Node, R])), {500, #{code => 'BAD_NODE', message => Message}}; From 104b1a63d99c0d49e76ca2e16cf46865c09e97b1 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 4 Nov 2022 17:13:54 +0100 Subject: [PATCH 138/165] test: pass down Opts in request_api/X --- .../test/emqx_mgmt_api_test_util.erl | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index 1bc29dfee..aed28930b 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -44,15 +44,20 @@ set_special_configs(_App) -> ok. request_api(Method, Url) -> - request_api(Method, Url, [], auth_header_(), []). + request_api(Method, Url, [], [], [], #{}). request_api(Method, Url, AuthOrHeaders) -> - request_api(Method, Url, [], AuthOrHeaders, []). + request_api(Method, Url, [], AuthOrHeaders, [], #{}). request_api(Method, Url, QueryParams, AuthOrHeaders) -> - request_api(Method, Url, QueryParams, AuthOrHeaders, []). + request_api(Method, Url, QueryParams, AuthOrHeaders, [], #{}). -request_api(Method, Url, QueryParams, AuthOrHeaders, []) when +request_api(Method, Url, QueryParams, AuthOrHeaders, Body) -> + request_api(Method, Url, QueryParams, AuthOrHeaders, Body, #{}). + +request_api(Method, Url, QueryParams, [], Body, Opts) -> + request_api(Method, Url, QueryParams, auth_header_(), Body, Opts); +request_api(Method, Url, QueryParams, AuthOrHeaders, [], Opts) when (Method =:= options) orelse (Method =:= get) orelse (Method =:= put) orelse @@ -65,10 +70,7 @@ request_api(Method, Url, QueryParams, AuthOrHeaders, []) when "" -> Url; _ -> Url ++ "?" ++ QueryParams end, - do_request_api(Method, {NewUrl, build_http_header(AuthOrHeaders)}, #{}); -request_api(Method, Url, QueryParams, AuthOrHeaders, Body) -> - request_api(Method, Url, QueryParams, AuthOrHeaders, Body, #{}). - + do_request_api(Method, {NewUrl, build_http_header(AuthOrHeaders)}, Opts); request_api(Method, Url, QueryParams, AuthOrHeaders, Body, Opts) when (Method =:= post) orelse (Method =:= patch) orelse From 1fb441dd9e1977ddaada4704fc9117b1f1cbc04f Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 4 Nov 2022 16:07:02 +0100 Subject: [PATCH 139/165] test: add tests for 'configs?node' --- .../test/emqx_mgmt_api_configs_SUITE.erl | 56 ++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 97939bbaf..83f68c5fe 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -30,6 +30,16 @@ init_per_suite(Config) -> end_per_suite(_) -> emqx_mgmt_api_test_util:end_suite([emqx_conf]). +init_per_testcase(TestCase = t_configs_node, Config) -> + ?MODULE:TestCase({'init', Config}); +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(TestCase = t_configs_node, Config) -> + ?MODULE:TestCase({'end', Config}); +end_per_testcase(_TestCase, Config) -> + Config. + t_get(_Config) -> {ok, Configs} = get_configs(), maps:map( @@ -188,6 +198,37 @@ t_dashboard(_Config) -> timer:sleep(1000), ok. +t_configs_node({'init', Config}) -> + Node = node(), + meck:expect(mria_mnesia, running_nodes, fun() -> [Node, bad_node, other_node] end), + meck:expect( + emqx_management_proto_v2, + get_full_config, + fun + (Node0) when Node0 =:= Node -> <<"\"self\"">>; + (other_node) -> <<"\"other\"">>; + (bad_node) -> {badrpc, bad} + end + ), + Config; +t_configs_node({'end', _}) -> + meck:unload([mria_mnesia, emqx_management_proto_v2]); +t_configs_node(_) -> + Node = atom_to_list(node()), + + ?assertEqual({ok, <<"self">>}, get_configs(Node, #{return_body => true})), + ?assertEqual({ok, <<"other">>}, get_configs("other_node", #{return_body => true})), + + {ExpType, ExpRes} = get_configs("unknown_node", #{return_body => true}), + ?assertEqual(error, ExpType), + ?assertMatch({{_, 404, _}, _, _}, ExpRes), + {_, _, Body} = ExpRes, + ?assertMatch(#{<<"code">> := <<"NOT_FOUND">>}, emqx_json:decode(Body, [return_maps])), + + ?assertMatch({error, {_, 500, _}}, get_configs("bad_node")). + +%% Helpers + get_config(Name) -> Path = emqx_mgmt_api_test_util:api_path(["configs", Name]), case emqx_mgmt_api_test_util:request_api(get, Path) of @@ -198,8 +239,19 @@ get_config(Name) -> end. get_configs() -> - Path = emqx_mgmt_api_test_util:api_path(["configs"]), - case emqx_mgmt_api_test_util:request_api(get, Path) of + get_configs([], #{}). + +get_configs(Node) -> + get_configs(Node, #{}). + +get_configs(Node, Opts) -> + Path = + case Node of + [] -> ["configs"]; + _ -> ["configs?node=" ++ Node] + end, + URI = emqx_mgmt_api_test_util:api_path(Path), + case emqx_mgmt_api_test_util:request_api(get, URI, [], [], [], Opts) of {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; Error -> Error end. From ea5b2cba6a81a93b1a4a6d5b57e9daa3d7a78abc Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 7 Nov 2022 15:02:26 +0100 Subject: [PATCH 140/165] chore: add changelog --- changes/v5.0.10-en.md | 2 ++ changes/v5.0.10-zh.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index 9ec10d3a7..69fee3aae 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -35,3 +35,5 @@ - Fix incorrect topic authorize checking of delayed messages [#9290](https://github.com/emqx/emqx/pull/9290). Now will determine the actual topic of the delayed messages, e.g. `$delayed/1/t/foo` will be treated as `t/foo` in authorize checks. + +- Fix query string parameter 'node' to `/configs` resource being ignored, return 404 if node does not exist [9310](https://github.com/emqx/emqx/pull/9310/). diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index d7e758162..63a4a89c5 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -33,3 +33,5 @@ - 修复延迟消息的主题授权判断不正确的问题 [#9290](https://github.com/emqx/emqx/pull/9290)。 现在将会对延迟消息中的真实主题进行授权判断,比如,`$delayed/1/t/foo` 会被当作 `t/foo` 进行判断。 + +- 修复 `/configs` API 的 'node' 参数的问题,如果节点不存在,则返回 HTTP 状态码 404 From ebaba0c2b179a4a8005ede762faa1a1d1d91d782 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 8 Nov 2022 11:36:11 +0800 Subject: [PATCH 141/165] fix: Revert "feat(banned): clean retained/delayed data when client is banned" This reverts commit 69701ff578adf2085f32cd4af484e725ea704123. --- apps/emqx/i18n/emqx_schema_i18n.conf | 13 ---- apps/emqx/priv/bpapi.versions | 1 - apps/emqx/src/emqx_banned.erl | 54 ++++------------ apps/emqx/src/emqx_flapping.erl | 4 +- apps/emqx/src/emqx_schema.erl | 8 --- apps/emqx/test/emqx_flapping_SUITE.erl | 3 +- .../i18n/emqx_mgmt_api_banned_i18n.conf | 12 ---- .../src/emqx_mgmt_api_banned.erl | 10 +-- apps/emqx_modules/src/emqx_delayed.erl | 30 ++------- .../src/proto/emqx_delayed_proto_v2.erl | 47 -------------- apps/emqx_modules/test/emqx_delayed_SUITE.erl | 61 ------------------- apps/emqx_retainer/src/emqx_retainer.erl | 24 +------- .../src/emqx_retainer_mnesia.erl | 33 +++------- .../test/emqx_retainer_SUITE.erl | 60 ------------------ 14 files changed, 31 insertions(+), 329 deletions(-) delete mode 100644 apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 23784785a..714a08704 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -494,19 +494,6 @@ emqx_schema { } } - flapping_detect_clean_when_banned { - desc { - en: "Clean retained/delayed messages when client is banned.\n" - "Note: This may be expensive and only supports users banned by clientid." - zh: "当客户端被禁时删除其保留、延迟消息" - "注意: 这个操作开销可能较大,且只支持通过 clientid 封禁的用户数据。" - } - label: { - en: "Clean when banned" - zh: "被禁时清理消息" - } - } - persistent_session_store_enabled { desc { en: "Use the database to store information about persistent sessions.\n" diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 29872f143..9997055dc 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -9,7 +9,6 @@ {emqx_conf,2}. {emqx_dashboard,1}. {emqx_delayed,1}. -{emqx_delayed,2}. {emqx_exhook,1}. {emqx_gateway_api_listeners,1}. {emqx_gateway_cm,1}. diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl index d7443543f..cf81c735b 100644 --- a/apps/emqx/src/emqx_banned.erl +++ b/apps/emqx/src/emqx_banned.erl @@ -32,13 +32,11 @@ -export([ check/1, create/1, - create/2, look_up/1, delete/1, info/1, format/1, - parse/1, - parse_opts/1 + parse/1 ]). %% gen_server callbacks @@ -65,13 +63,6 @@ -compile(nowarn_export_all). -endif. --type banned_opts() :: #{ - clean => boolean(), - atom() => term() -}. - --export_type([banned_opts/0]). - %%-------------------------------------------------------------------- %% Mnesia bootstrap %%-------------------------------------------------------------------- @@ -150,11 +141,6 @@ parse(Params) -> {error, ErrorReason} end end. - -parse_opts(Params) -> - Clean = maps:get(<<"clean">>, Params, false), - #{clean => Clean}. - pares_who(#{as := As, who := Who}) -> pares_who(#{<<"as">> => As, <<"who">> => Who}); pares_who(#{<<"as">> := peerhost, <<"who">> := Peerhost0}) -> @@ -176,15 +162,13 @@ to_rfc3339(Timestamp) -> -spec create(emqx_types:banned() | map()) -> {ok, emqx_types:banned()} | {error, {already_exist, emqx_types:banned()}}. -create( - #{ - who := Who, - by := By, - reason := Reason, - at := At, - until := Until - } = Data -) -> +create(#{ + who := Who, + by := By, + reason := Reason, + at := At, + until := Until +}) -> Banned = #banned{ who = Who, by = By, @@ -192,16 +176,11 @@ create( at = At, until = Until }, - create(Banned, Data); -create(Banned = #banned{}) -> - create(Banned, #{clean => false}). - --spec create(emqx_types:banned(), banned_opts()) -> - {ok, emqx_types:banned()} | {error, {already_exist, emqx_types:banned()}}. -create(Banned = #banned{who = Who}, Opts) -> + create(Banned); +create(Banned = #banned{who = Who}) -> case look_up(Who) of [] -> - insert_banned(Banned, Opts), + mria:dirty_write(?BANNED_TAB, Banned), {ok, Banned}; [OldBanned = #banned{until = Until}] -> %% Don't support shorten or extend the until time by overwrite. @@ -211,7 +190,7 @@ create(Banned = #banned{who = Who}, Opts) -> {error, {already_exist, OldBanned}}; %% overwrite expired one is ok. false -> - insert_banned(Banned, Opts), + mria:dirty_write(?BANNED_TAB, Banned), {ok, Banned} end end. @@ -287,12 +266,3 @@ expire_banned_items(Now) -> ok, ?BANNED_TAB ). - -insert_banned(Banned, Opts) -> - mria:dirty_write(?BANNED_TAB, Banned), - run_hooks(Banned, Opts). - -run_hooks(Banned, #{clean := true}) -> - emqx_hooks:run('client.banned', [Banned]); -run_hooks(_Banned, _Opts) -> - ok. diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index f0492cbf9..7e72c488f 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -121,7 +121,7 @@ handle_cast( started_at = StartedAt, detect_cnt = DetectCnt }, - #{window_time := WindTime, ban_time := Interval, clean_when_banned := Clean}}, + #{window_time := WindTime, ban_time := Interval}}, State ) -> case now_diff(StartedAt) < WindTime of @@ -145,7 +145,7 @@ handle_cast( at = Now, until = Now + (Interval div 1000) }, - {ok, _} = emqx_banned:create(Banned, #{clean => Clean}), + {ok, _} = emqx_banned:create(Banned), ok; false -> ?SLOG( diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 74ed51a38..4c26f86f9 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -640,14 +640,6 @@ fields("flapping_detect") -> default => "5m", desc => ?DESC(flapping_detect_ban_time) } - )}, - {"clean_when_banned", - sc( - boolean(), - #{ - default => false, - desc => ?DESC(flapping_detect_clean_when_banned) - } )} ]; fields("force_shutdown") -> diff --git a/apps/emqx/test/emqx_flapping_SUITE.erl b/apps/emqx/test/emqx_flapping_SUITE.erl index 774ccaae2..f37e20fdc 100644 --- a/apps/emqx/test/emqx_flapping_SUITE.erl +++ b/apps/emqx/test/emqx_flapping_SUITE.erl @@ -34,8 +34,7 @@ init_per_suite(Config) -> % 0.1s window_time => 100, %% 2s - ban_time => 2000, - clean_when_banned => false + ban_time => 2000 } ), Config. diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf index 04b96891c..3045cb293 100644 --- a/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf +++ b/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf @@ -95,16 +95,4 @@ emqx_mgmt_api_banned { zh: """封禁结束时间""" } } - clean { - desc { - en: """Clean retained/delayed messages when client is banned.""" - """Note: This may be expensive and only supports users banned by clientid.""" - zh: """当客户端被禁时删除其保留、延迟消息""" - """注意: 这个操作开销可能较大,且只支持通过 clientid 封禁的用户数据。""" - } - label { - en: """Clean when banned""" - zh: """被禁时清理消息""" - } - } } diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 1eacc67c2..2eb8908c6 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -150,13 +150,6 @@ fields(ban) -> desc => ?DESC(until), required => false, example => <<"2021-10-25T21:53:47+08:00">> - })}, - {clean, - hoconsc:mk(boolean(), #{ - desc => ?DESC(clean), - required => false, - default => false, - example => false })} ]. @@ -168,8 +161,7 @@ banned(post, #{body := Body}) -> {error, Reason} -> {400, 'BAD_REQUEST', list_to_binary(Reason)}; Ban -> - Opts = emqx_banned:parse_opts(Body), - case emqx_banned:create(Ban, Opts) of + case emqx_banned:create(Ban) of {ok, Banned} -> {200, format(Banned)}; {error, {already_exist, Old}} -> diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 6a8b8de5b..76646bc64 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -23,7 +23,6 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). --include_lib("stdlib/include/ms_transform.hrl"). %% Mnesia bootstrap -export([mnesia/1]). @@ -32,8 +31,7 @@ -export([ start_link/0, - on_message_publish/1, - on_client_banned/1 + on_message_publish/1 ]). %% gen_server callbacks @@ -46,7 +44,7 @@ code_change/3 ]). -%% API +%% gen_server callbacks -export([ load/0, unload/0, @@ -59,9 +57,7 @@ delete_delayed_message/1, delete_delayed_message/2, cluster_list/1, - cluster_query/4, - clean_by_clientid/1, - do_clean_by_clientid/1 + cluster_query/4 ]). -export([ @@ -142,11 +138,6 @@ on_message_publish( on_message_publish(Msg) -> {ok, Msg}. -on_client_banned(#banned{who = {clientid, ClientId}}) -> - clean_by_clientid(ClientId); -on_client_banned(_) -> - ok. - %%-------------------------------------------------------------------- %% Start delayed publish server %%-------------------------------------------------------------------- @@ -237,7 +228,7 @@ get_delayed_message(Id) -> get_delayed_message(Node, Id) when Node =:= node() -> get_delayed_message(Id); get_delayed_message(Node, Id) -> - emqx_delayed_proto_v2:get_delayed_message(Node, Id). + emqx_delayed_proto_v1:get_delayed_message(Node, Id). -spec delete_delayed_message(binary()) -> with_id_return(). delete_delayed_message(Id) -> @@ -252,7 +243,7 @@ delete_delayed_message(Id) -> delete_delayed_message(Node, Id) when Node =:= node() -> delete_delayed_message(Id); delete_delayed_message(Node, Id) -> - emqx_delayed_proto_v2:delete_delayed_message(Node, Id). + emqx_delayed_proto_v1:delete_delayed_message(Node, Id). update_config(Config) -> emqx_conf:update([delayed], Config, #{rawconf_with_defaults => true, override_to => cluster}). @@ -261,15 +252,6 @@ post_config_update(_KeyPath, _ConfigReq, NewConf, _OldConf, _AppEnvs) -> Enable = maps:get(enable, NewConf, undefined), load_or_unload(Enable). -clean_by_clientid(ClientId) -> - Nodes = mria_mnesia:running_nodes(), - emqx_delayed_proto_v2:clean_by_clientid(Nodes, ClientId). - -do_clean_by_clientid(ClientId) -> - ets:select_delete( - ?TAB, ets:fun2ms(fun(#delayed_message{msg = Msg}) -> Msg#message.from =:= ClientId end) - ). - %%-------------------------------------------------------------------- %% gen_server callback %%-------------------------------------------------------------------- @@ -401,11 +383,9 @@ delayed_count() -> mnesia:table_info(?TAB, size). do_load_or_unload(true, State) -> emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_DELAY_PUB), - ok = emqx_hooks:put('client.banned', {?MODULE, on_client_banned, []}, ?HP_LOWEST), State; do_load_or_unload(false, #{publish_timer := PubTimer} = State) -> emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), - ok = emqx_hooks:del('client.banned', {?MODULE, on_client_banned}), emqx_misc:cancel_timer(PubTimer), ets:delete_all_objects(?TAB), State#{publish_timer := undefined, publish_at := 0}; diff --git a/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl b/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl deleted file mode 100644 index 9bbf35720..000000000 --- a/apps/emqx_modules/src/proto/emqx_delayed_proto_v2.erl +++ /dev/null @@ -1,47 +0,0 @@ -%%-------------------------------------------------------------------- -%%Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_delayed_proto_v2). - --behaviour(emqx_bpapi). - --export([ - introduced_in/0, - get_delayed_message/2, - delete_delayed_message/2, - clean_by_clientid/2 -]). - --include_lib("emqx/include/bpapi.hrl"). - --define(TIMEOUT, 15000). - -introduced_in() -> - "5.0.10". - --spec get_delayed_message(node(), binary()) -> - emqx_delayed:with_id_return(map()) | emqx_rpc:badrpc(). -get_delayed_message(Node, Id) -> - rpc:call(Node, emqx_delayed, get_delayed_message, [Id]). - --spec delete_delayed_message(node(), binary()) -> emqx_delayed:with_id_return() | emqx_rpc:badrpc(). -delete_delayed_message(Node, Id) -> - rpc:call(Node, emqx_delayed, delete_delayed_message, [Id]). - --spec clean_by_clientid(list(node()), emqx_types:clientid()) -> - emqx_rpc:erpc_multicall(). -clean_by_clientid(Nodes, ClientID) -> - erpc:multicall(Nodes, emqx_delayed, do_clean_by_clientid, [ClientID], ?TIMEOUT). diff --git a/apps/emqx_modules/test/emqx_delayed_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_SUITE.erl index bd1de3849..e5e3db98c 100644 --- a/apps/emqx_modules/test/emqx_delayed_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_SUITE.erl @@ -212,67 +212,6 @@ t_delayed_precision(_) -> _ = on_message_publish(DelayedMsg0), ?assert(FutureDiff() =< MaxSpan). -t_banned_clean(_) -> - emqx:update_config([delayed, max_delayed_messages], 10000), - ClientId1 = <<"bc1">>, - ClientId2 = <<"bc2">>, - {ok, C1} = emqtt:start_link([{clientid, ClientId1}, {clean_start, true}, {proto_ver, v5}]), - {ok, _} = emqtt:connect(C1), - - {ok, C2} = emqtt:start_link([{clientid, ClientId2}, {clean_start, true}, {proto_ver, v5}]), - {ok, _} = emqtt:connect(C2), - - [ - begin - emqtt:publish( - Conn, - <<"$delayed/60/0/", ClientId/binary>>, - <<"">>, - [{qos, 0}, {retain, false}] - ), - emqtt:publish( - Conn, - <<"$delayed/60/1/", ClientId/binary>>, - <<"">>, - [{qos, 0}, {retain, false}] - ) - end - || {ClientId, Conn} <- lists:zip([ClientId1, ClientId2], [C1, C2]) - ], - - emqtt:publish( - C2, - <<"$delayed/60/2/", ClientId2/binary>>, - <<"">>, - [{qos, 0}, {retain, false}] - ), - - timer:sleep(500), - ?assertMatch(#{meta := #{count := 5}}, emqx_delayed:list(#{page => 1, limit => 10})), - - Now = erlang:system_time(second), - Who = {clientid, ClientId2}, - try - emqx_banned:create(#{ - who => Who, - by => <<"test">>, - reason => <<"test">>, - at => Now, - until => Now + 120, - clean => true - }), - - timer:sleep(500), - - ?assertMatch(#{meta := #{count := 2}}, emqx_delayed:list(#{page => 1, limit => 10})) - after - emqx_banned:delete(Who), - emqx_delayed:clean_by_clientid(ClientId1) - end, - timer:sleep(500), - ok = emqtt:disconnect(C1), - ok = emqtt:disconnect(C2). - subscribe_proc() -> Self = self(), Ref = erlang:make_ref(), diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index d6a025a41..5d911b5f4 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -19,7 +19,6 @@ -behaviour(gen_server). -include("emqx_retainer.hrl"). --include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). @@ -27,8 +26,7 @@ -export([ on_session_subscribed/4, - on_message_publish/2, - on_client_banned/1 + on_message_publish/2 ]). -export([ @@ -41,7 +39,6 @@ get_expiry_time/1, update_config/1, clean/0, - clean_by_clientid/1, delete/1, page_read/3, post_config_update/5, @@ -83,7 +80,6 @@ -callback match_messages(context(), topic(), cursor()) -> {ok, list(), cursor()}. -callback clear_expired(context()) -> ok. -callback clean(context()) -> ok. --callback clean_by_clientid(context(), emqx_types:clientid()) -> ok. -callback size(context()) -> non_neg_integer(). %%-------------------------------------------------------------------- @@ -122,11 +118,6 @@ on_message_publish(Msg = #message{flags = #{retain := true}}, Context) -> on_message_publish(Msg, _) -> {ok, Msg}. -on_client_banned(#banned{who = {clientid, ClientId}}) -> - clean_by_clientid(ClientId); -on_client_banned(_) -> - ok. - %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -160,9 +151,6 @@ update_config(Conf) -> clean() -> call(?FUNCTION_NAME). -clean_by_clientid(ClientId) -> - call({?FUNCTION_NAME, ClientId}). - delete(Topic) -> call({?FUNCTION_NAME, Topic}). @@ -219,9 +207,6 @@ handle_call({update_config, NewConf, OldConf}, _, State) -> handle_call(clean, _, #{context := Context} = State) -> clean(Context), {reply, ok, State}; -handle_call({clean_by_clientid, ClientId}, _, #{context := Context} = State) -> - clean_by_clientid(Context, ClientId), - {reply, ok, State}; handle_call({delete, Topic}, _, #{context := Context} = State) -> delete_message(Context, Topic), {reply, ok, State}; @@ -313,11 +298,6 @@ clean(Context) -> Mod = get_backend_module(), Mod:clean(Context). --spec clean_by_clientid(context(), emqx_types:clientid()) -> ok. -clean_by_clientid(Context, ClientId) -> - Mod = get_backend_module(), - Mod:clean_by_clientid(Context, ClientId). - -spec update_config(state(), hocons:config(), hocons:config()) -> state(). update_config(State, Conf, OldConf) -> update_config( @@ -453,13 +433,11 @@ load(Context) -> 'session.subscribed', {?MODULE, on_session_subscribed, [Context]}, ?HP_RETAINER ), ok = emqx_hooks:put('message.publish', {?MODULE, on_message_publish, [Context]}, ?HP_RETAINER), - ok = emqx_hooks:put('client.banned', {?MODULE, on_client_banned, []}, ?HP_LOWEST), emqx_stats:update_interval(emqx_retainer_stats, fun ?MODULE:stats_fun/0), ok. unload() -> ok = emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), ok = emqx_hooks:del('session.subscribed', {?MODULE, on_session_subscribed}), - ok = emqx_hooks:del('client.banned', {?MODULE, on_client_banned}), emqx_stats:cancel_update(emqx_retainer_stats), ok. diff --git a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl index 281d9c3ce..c236b9c28 100644 --- a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl +++ b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl @@ -33,14 +33,13 @@ match_messages/3, clear_expired/1, clean/1, - clean_by_clientid/2, size/1 ]). %% Internal exports (RPC) -export([ do_store_retained/1, - do_clear/1, + do_clear_expired/0, do_delete_message/1, do_populate_index_meta/1, do_reindex_batch/2 @@ -62,8 +61,6 @@ -record(retained_index, {key, expiry_time}). -record(retained_index_meta, {key, read_indices, write_indices, reindexing, extra}). --type retained_message() :: #retained_message{}. - -define(META_KEY, index_meta). -define(CLEAR_BATCH_SIZE, 1000). @@ -167,22 +164,18 @@ do_store_retained(#message{topic = Topic} = Msg) -> end. clear_expired(_) -> - NowMs = erlang:system_time(millisecond), - {atomic, _} = mria:transaction(?RETAINER_SHARD, fun ?MODULE:do_clear/1, [ - fun( - #retained_message{expiry_time = ExpiryTime} - ) -> - (ExpiryTime =/= 0) and (ExpiryTime < NowMs) - end - ]), + {atomic, _} = mria:transaction(?RETAINER_SHARD, fun ?MODULE:do_clear_expired/0), ok. --spec do_clear(fun((retained_message()) -> boolean())) -> ok. -do_clear(Pred) -> +do_clear_expired() -> + NowMs = erlang:system_time(millisecond), QH = qlc:q([ TopicTokens - || #retained_message{topic = TopicTokens} = Data <- mnesia:table(?TAB_MESSAGE, [{lock, write}]), - Pred(Data) + || #retained_message{ + topic = TopicTokens, + expiry_time = ExpiryTime + } <- mnesia:table(?TAB_MESSAGE, [{lock, write}]), + (ExpiryTime =/= 0) and (ExpiryTime < NowMs) ]), QC = qlc:cursor(QH), clear_batch(db_indices(write), QC). @@ -270,14 +263,6 @@ clean(_) -> _ = mria:clear_table(?TAB_INDEX), ok. -clean_by_clientid(_, ClientId) -> - {atomic, _} = mria:transaction(?RETAINER_SHARD, fun ?MODULE:do_clear/1, [ - fun(Msg) -> - Msg#retained_message.msg#message.from =:= ClientId - end - ]), - ok. - size(_) -> table_size(). diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index 23d4aee98..09e6c4bb4 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -626,66 +626,6 @@ t_get_basic_usage_info(_Config) -> ?assertEqual(#{retained_messages => 5}, emqx_retainer:get_basic_usage_info()), ok. -t_banned_clean(_) -> - ClientId1 = <<"bc1">>, - ClientId2 = <<"bc2">>, - {ok, C1} = emqtt:start_link([{clientid, ClientId1}, {clean_start, true}, {proto_ver, v5}]), - {ok, _} = emqtt:connect(C1), - - {ok, C2} = emqtt:start_link([{clientid, ClientId2}, {clean_start, true}, {proto_ver, v5}]), - {ok, _} = emqtt:connect(C2), - - [ - begin - emqtt:publish( - Conn, - <<"bc/0/", ClientId/binary>>, - <<"this is a retained message 0">>, - [{qos, 0}, {retain, true}] - ), - emqtt:publish( - Conn, - <<"bc/1/", ClientId/binary>>, - <<"this is a retained message 1">>, - [{qos, 0}, {retain, true}] - ) - end - || {ClientId, Conn} <- lists:zip([ClientId1, ClientId2], [C1, C2]) - ], - - emqtt:publish( - C2, - <<"bc/2/", ClientId2/binary>>, - <<"this is a retained message 2">>, - [{qos, 0}, {retain, true}] - ), - - timer:sleep(500), - {ok, List} = emqx_retainer:page_read(<<"bc/+/+">>, 1, 10), - ?assertEqual(5, length(List)), - - Now = erlang:system_time(second), - Who = {clientid, ClientId2}, - emqx_banned:create(#{ - who => Who, - by => <<"test">>, - reason => <<"test">>, - at => Now, - until => Now + 120, - clean => true - }), - - timer:sleep(500), - - {ok, List2} = emqx_retainer:page_read(<<"bc/#">>, 1, 10), - ?assertEqual(2, length(List2)), - - emqx_banned:delete(Who), - emqx_retainer:clean(), - timer:sleep(500), - ok = emqtt:disconnect(C1), - ok = emqtt:disconnect(C2). - %% test whether the app can start normally after disabling emqx_retainer %% fix: https://github.com/emqx/emqx/pull/8911 test_disable_then_start(_Config) -> From dd7d4224cec928aeb62b6be2d9efd2eafd3620e7 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 8 Nov 2022 16:32:24 +0800 Subject: [PATCH 142/165] feat(delayed): check if the source client is banned when publishing a delayed message --- apps/emqx_modules/src/emqx_delayed.erl | 19 ++++++++-- apps/emqx_modules/test/emqx_delayed_SUITE.erl | 36 ++++++++++++++++++- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 76646bc64..f511d74d9 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -373,8 +373,23 @@ do_publish({Ts, _Id}, Now, Acc) when Ts > Now -> Acc; do_publish(Key = {Ts, _Id}, Now, Acc) when Ts =< Now -> case mnesia:dirty_read(?TAB, Key) of - [] -> ok; - [#delayed_message{msg = Msg}] -> emqx_pool:async_submit(fun emqx:publish/1, [Msg]) + [] -> + ok; + [#delayed_message{msg = Msg}] -> + case emqx_banned:look_up({clientid, Msg#message.from}) of + [] -> + emqx_pool:async_submit(fun emqx:publish/1, [Msg]); + _ -> + ?tp( + notice, + ignore_delayed_message_publish, + #{ + reason => "client is banned", + clienid => Msg#message.from + } + ), + ok + end end, do_publish(mnesia:dirty_next(?TAB, Key), Now, [Key | Acc]). diff --git a/apps/emqx_modules/test/emqx_delayed_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_SUITE.erl index e5e3db98c..5864646ad 100644 --- a/apps/emqx_modules/test/emqx_delayed_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_SUITE.erl @@ -26,6 +26,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). %%-------------------------------------------------------------------- %% Setups @@ -36,7 +37,8 @@ }). all() -> - emqx_common_test_helpers:all(?MODULE). + [t_banned_delayed]. +%% emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF, #{ @@ -212,6 +214,38 @@ t_delayed_precision(_) -> _ = on_message_publish(DelayedMsg0), ?assert(FutureDiff() =< MaxSpan). +t_banned_delayed(_) -> + emqx:update_config([delayed, max_delayed_messages], 10000), + ClientId1 = <<"bc1">>, + ClientId2 = <<"bc2">>, + + Now = erlang:system_time(second), + Who = {clientid, ClientId2}, + emqx_banned:create(#{ + who => Who, + by => <<"test">>, + reason => <<"test">>, + at => Now, + until => Now + 120 + }), + + snabbkaffe:start_trace(), + lists:foreach( + fun(ClientId) -> + Msg = emqx_message:make(ClientId, <<"$delayed/1/bc">>, <<"payload">>), + emqx_delayed:on_message_publish(Msg) + end, + [ClientId1, ClientId1, ClientId1, ClientId2, ClientId2] + ), + + timer:sleep(2000), + Trace = snabbkaffe:collect_trace(), + snabbkaffe:stop(), + emqx_banned:delete(Who), + mnesia:clear_table(emqx_delayed), + + ?assertEqual(2, length(?of_kind(ignore_delayed_message_publish, Trace))). + subscribe_proc() -> Self = self(), Ref = erlang:make_ref(), From 848b68de887fd9dfa85775bb2c0fa3503f66235d Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 8 Nov 2022 17:04:55 +0800 Subject: [PATCH 143/165] chore: update changes --- changes/v5.0.10-en.md | 3 ++- changes/v5.0.10-zh.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index 9ec10d3a7..3f679e965 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -4,7 +4,8 @@ - Improve `/nodes` API responsiveness [#9221](https://github.com/emqx/emqx/pull/9221). -- Allow clear retained/delayed data when client is banned [#9139](https://github.com/emqx/emqx/pull/9139). +- Improve the integration of the `banned` and the `delayed` feature [#9326](https://github.com/emqx/emqx/pull/9326). + Now when publishing a delayed message will check first if its source client is banned, if true, this publish will be ignored. - Update `gen_rpc` library to version 3.0 [#9187](https://github.com/emqx/emqx/pull/9187). diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index d7e758162..3f83c913e 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -4,7 +4,8 @@ - 提升 `/nodes` API 响应速度 [#9221](https://github.com/emqx/emqx/pull/9221)。 -- 支持拉黑客户端并从数据库中删除保留和延迟发布的消息 [#9139](https://github.com/emqx/emqx/pull/9139)。 +- 增强 `封禁` 和 `延迟消息` 这两个功能的集成性 [#9326](https://github.com/emqx/emqx/pull/9326)。 + 现在发送延迟消息前,会先检查消息的来源客户端是否被封禁了,如果是,这条延迟消息将会被忽略。 - 升级 `gen_rpc` 库到 3.0 [#9187](https://github.com/emqx/emqx/pull/9187)。 From 62856a8b77a35ec6811019f79ae789c2fa5fd121 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 8 Nov 2022 16:10:18 +0100 Subject: [PATCH 144/165] style: fix typo in changelog Co-authored-by: Thales Macedo Garitezi --- changes/v5.0.10-en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index 69fee3aae..a4e2097b2 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -36,4 +36,4 @@ - Fix incorrect topic authorize checking of delayed messages [#9290](https://github.com/emqx/emqx/pull/9290). Now will determine the actual topic of the delayed messages, e.g. `$delayed/1/t/foo` will be treated as `t/foo` in authorize checks. -- Fix query string parameter 'node' to `/configs` resource being ignored, return 404 if node does not exist [9310](https://github.com/emqx/emqx/pull/9310/). +- Fix query string parameter 'node' to `/configs` resource being ignored, return 404 if node does not exist [#9310](https://github.com/emqx/emqx/pull/9310/). From 2cd457e07c29af3576bee1c08eae66422f26eced Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 7 Nov 2022 14:19:26 +0800 Subject: [PATCH 145/165] fix(channel): session takenover & discarded maybe call disconnect hook --- apps/emqx/src/emqx_channel.erl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 494c77ec7..ea35abfba 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -2098,7 +2098,7 @@ parse_topic_filters(TopicFilters) -> lists:map(fun emqx_topic:parse/1, TopicFilters). %%-------------------------------------------------------------------- -%% Ensure disconnected +%% Maybe & Ensure disconnected ensure_disconnected( Reason, @@ -2205,6 +2205,7 @@ shutdown(success, Reply, Packet, Channel) -> shutdown(Reason, Reply, Packet, Channel) -> {shutdown, Reason, Reply, Packet, Channel}. +%% mqtt v5 connected sessions disconnect_and_shutdown( Reason, Reply, @@ -2214,9 +2215,12 @@ disconnect_and_shutdown( ) when ConnState =:= connected orelse ConnState =:= reauthenticating -> - shutdown(Reason, Reply, ?DISCONNECT_PACKET(reason_code(Reason)), Channel); + NChannel = ensure_disconnected(Reason, Channel), + shutdown(Reason, Reply, ?DISCONNECT_PACKET(reason_code(Reason)), NChannel); +%% mqtt v3/v4 sessions, mqtt v5 other conn_state sessions disconnect_and_shutdown(Reason, Reply, Channel) -> - shutdown(Reason, Reply, Channel). + NChannel = ensure_disconnected(Reason, Channel), + shutdown(Reason, Reply, NChannel). sp(true) -> 1; sp(false) -> 0. From 4f002066937dbf54acebfe4b1175afa813f42ec0 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 8 Nov 2022 22:54:37 +0800 Subject: [PATCH 146/165] test: different resons for disconnect event --- .../test/emqx_rule_engine_SUITE.erl | 167 +++++++++++++++++- 1 file changed, 166 insertions(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 4e0c7dfe3..17fe1a36c 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -91,7 +91,13 @@ groups() -> t_sqlparse_new_map, t_sqlparse_invalid_json ]}, - {events, [], [t_events]}, + {events, [], [ + t_events, + t_event_client_disconnected_normal, + t_event_client_disconnected_kicked, + t_event_client_disconnected_discarded, + t_event_client_disconnected_takenover + ]}, {telemetry, [], [ t_get_basic_usage_info_0, t_get_basic_usage_info_1 @@ -474,6 +480,165 @@ t_events(_Config) -> client_connack_failed(), ok. +t_event_client_disconnected_normal(_Config) -> + SQL = + "select * " + "from \"$events/client_disconnected\" ", + RepubT = <<"repub/to/disconnected/normal">>, + + {ok, TopicRule} = emqx_rule_engine:create_rule( + #{ + sql => SQL, + id => ?TMP_RULEID, + actions => [republish_action(RepubT, <<>>)] + } + ), + + {ok, Client} = emqtt:start_link([{clientid, <<"get_repub_client">>}, {username, <<"emqx0">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, RepubT, 0), + ct:sleep(200), + {ok, Client1} = emqtt:start_link([{clientid, <<"emqx">>}, {username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client1), + emqtt:disconnect(Client1), + + receive + {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(RepubT, T), + ?assertMatch(#{<<"reason">> := <<"normal">>}, emqx_json:decode(Payload, [return_maps])) + after 1000 -> + ct:fail(wait_for_repub_disconnected_normal) + end, + emqtt:stop(Client), + + delete_rule(TopicRule). + +t_event_client_disconnected_kicked(_Config) -> + SQL = + "select * " + "from \"$events/client_disconnected\" ", + RepubT = <<"repub/to/disconnected/kicked">>, + + {ok, TopicRule} = emqx_rule_engine:create_rule( + #{ + sql => SQL, + id => ?TMP_RULEID, + actions => [republish_action(RepubT, <<>>)] + } + ), + + {ok, Client} = emqtt:start_link([{clientid, <<"get_repub_client">>}, {username, <<"emqx0">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, RepubT, 0), + ct:sleep(200), + + {ok, Client1} = emqtt:start_link([{clientid, <<"emqx">>}, {username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client1), + %% the process will receive {'EXIT',{shutdown,tcp_closed}} + unlink(Client1), + + emqx_cm:kick_session(<<"emqx">>), + + receive + {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(RepubT, T), + ?assertMatch(#{<<"reason">> := <<"kicked">>}, emqx_json:decode(Payload, [return_maps])) + after 1000 -> + ct:fail(wait_for_repub_disconnected_kicked) + end, + + emqtt:stop(Client), + delete_rule(TopicRule). + +t_event_client_disconnected_discarded(_Config) -> + SQL = + "select * " + "from \"$events/client_disconnected\" ", + RepubT = <<"repub/to/disconnected/discarded">>, + + {ok, TopicRule} = emqx_rule_engine:create_rule( + #{ + sql => SQL, + id => ?TMP_RULEID, + actions => [republish_action(RepubT, <<>>)] + } + ), + + {ok, Client} = emqtt:start_link([{clientid, <<"get_repub_client">>}, {username, <<"emqx0">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, RepubT, 0), + ct:sleep(200), + + {ok, Client1} = emqtt:start_link([{clientid, <<"emqx">>}, {username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client1), + %% the process will receive {'EXIT',{shutdown,tcp_closed}} + unlink(Client1), + + {ok, Client2} = emqtt:start_link([ + {clientid, <<"emqx">>}, {username, <<"emqx">>}, {clean_start, true} + ]), + {ok, _} = emqtt:connect(Client2), + + receive + {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(RepubT, T), + ?assertMatch( + #{<<"reason">> := <<"discarded">>}, emqx_json:decode(Payload, [return_maps]) + ) + after 1000 -> + ct:fail(wait_for_repub_disconnected_discarded) + end, + emqtt:stop(Client), + emqtt:stop(Client2), + + delete_rule(TopicRule). + +t_event_client_disconnected_takenover(_Config) -> + SQL = + "select * " + "from \"$events/client_disconnected\" ", + RepubT = <<"repub/to/disconnected/takenover">>, + + {ok, TopicRule} = emqx_rule_engine:create_rule( + #{ + sql => SQL, + id => ?TMP_RULEID, + actions => [republish_action(RepubT, <<>>)] + } + ), + + {ok, ClientRecv} = emqtt:start_link([ + {clientid, <<"get_repub_client">>}, {username, <<"emqx0">>} + ]), + {ok, _} = emqtt:connect(ClientRecv), + {ok, _, _} = emqtt:subscribe(ClientRecv, RepubT, 0), + ct:sleep(200), + + {ok, Client1} = emqtt:start_link([{clientid, <<"emqx">>}, {username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client1), + %% the process will receive {'EXIT',{shutdown,tcp_closed}} + unlink(Client1), + + {ok, Client2} = emqtt:start_link([ + {clientid, <<"emqx">>}, {username, <<"emqx">>}, {clean_start, false} + ]), + {ok, _} = emqtt:connect(Client2), + + receive + {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(RepubT, T), + ?assertMatch( + #{<<"reason">> := <<"takenover">>}, emqx_json:decode(Payload, [return_maps]) + ) + after 1000 -> + ct:fail(wait_for_repub_disconnected_discarded) + end, + + emqtt:stop(ClientRecv), + emqtt:stop(Client2), + + delete_rule(TopicRule). + client_connack_failed() -> {ok, Client} = emqtt:start_link( [ From e0ed0855ffcd5318af334f8775ef729cbfd6b8a0 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 2 Nov 2022 16:10:21 +0100 Subject: [PATCH 147/165] fix(emqx_authz_api_sources): return 'code' in response body for 404 --- apps/emqx_authz/src/emqx_authz_api_sources.erl | 2 +- apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 9ff65f8a5..627a7f5e3 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -241,7 +241,7 @@ source(Method, #{bindings := #{type := Type} = Bindings} = Req) when source(get, #{bindings := #{type := Type}}) -> case get_raw_source(Type) of [] -> - {404, #{message => <<"Not found ", Type/binary>>}}; + {404, #{code => <<"NOT_FOUND">>, message => <<"Not found ", Type/binary>>}}; [#{<<"type">> := <<"file">>, <<"enable">> := Enable, <<"path">> := Path}] -> case file:read_file(Path) of {ok, Rules} -> diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index 3357798eb..11a610d2f 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -181,6 +181,12 @@ t_api(_) -> {ok, 200, Result1} = request(get, uri(["authorization", "sources"]), []), ?assertEqual([], get_sources(Result1)), + {ok, 404, ErrResult} = request(get, uri(["authorization", "sources", "http"]), []), + ?assertMatch( + #{<<"code">> := <<"NOT_FOUND">>, <<"message">> := <<"Not found http">>}, + jsx:decode(ErrResult) + ), + [ begin {ok, 204, _} = request(post, uri(["authorization", "sources"]), Source) From ba1e19f06835647b3ee065a747385d19e074e1de Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 3 Nov 2022 15:57:30 +0100 Subject: [PATCH 148/165] fix(emqx_authz_api_sources): make schema fit to what we send --- apps/emqx_authz/src/emqx_authz_api_sources.erl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 627a7f5e3..bfb7f908a 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -40,7 +40,8 @@ -export([ api_spec/0, paths/0, - schema/1 + schema/1, + fields/1 ]). -export([ @@ -63,6 +64,9 @@ paths() -> "/authorization/sources/:type/move" ]. +fields(sources) -> + [{sources, mk(array(hoconsc:union(authz_sources_type_refs())), #{desc => ?DESC(sources)})}]. + %%-------------------------------------------------------------------- %% Schema for each URI %%-------------------------------------------------------------------- @@ -75,10 +79,7 @@ schema("/authorization/sources") -> tags => ?TAGS, responses => #{ - 200 => mk( - array(hoconsc:union(authz_sources_type_refs())), - #{desc => ?DESC(sources)} - ) + 200 => ref(?MODULE, sources) } }, post => From 3eda28ba9cd0417fbf055c58f368ba022e610f08 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 3 Nov 2022 16:28:06 +0100 Subject: [PATCH 149/165] chore: add changelog --- changes/v5.0.10-en.md | 4 ++++ changes/v5.0.10-zh.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index 3f679e965..cdfb77d5d 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -36,3 +36,7 @@ - Fix incorrect topic authorize checking of delayed messages [#9290](https://github.com/emqx/emqx/pull/9290). Now will determine the actual topic of the delayed messages, e.g. `$delayed/1/t/foo` will be treated as `t/foo` in authorize checks. + +- Add property `code` to error response for `/authentication/sources/:type` [9299](https://github.com/emqx/emqx/pull/9299). + +- Align documentation for `/authentication/sources` with what we actually send [9299](https://github.com/emqx/emqx/pull/9299). diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 3f83c913e..614d9bd51 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -34,3 +34,7 @@ - 修复延迟消息的主题授权判断不正确的问题 [#9290](https://github.com/emqx/emqx/pull/9290)。 现在将会对延迟消息中的真实主题进行授权判断,比如,`$delayed/1/t/foo` 会被当作 `t/foo` 进行判断。 + +- 为 API `/authentication/sources/:type` 的返回值增加 `code` 字段 [9299](https://github.com/emqx/emqx/pull/9299)。 + +- 对其文档,`/authentication/sources` 接口的文档仅列出已经支持的资源 [9299](https://github.com/emqx/emqx/pull/9299)。 From 0678e05e840734d3e7576b155107a7a9db612824 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 8 Nov 2022 12:49:45 +0100 Subject: [PATCH 150/165] style: fix message returned for 404 Co-authored-by: Zaiming (Stone) Shi --- apps/emqx_authz/src/emqx_authz_api_sources.erl | 2 +- apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index bfb7f908a..3af2988db 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -242,7 +242,7 @@ source(Method, #{bindings := #{type := Type} = Bindings} = Req) when source(get, #{bindings := #{type := Type}}) -> case get_raw_source(Type) of [] -> - {404, #{code => <<"NOT_FOUND">>, message => <<"Not found ", Type/binary>>}}; + {404, #{code => <<"NOT_FOUND">>, message => <<"Not found: ", Type/binary>>}}; [#{<<"type">> := <<"file">>, <<"enable">> := Enable, <<"path">> := Path}] -> case file:read_file(Path) of {ok, Rules} -> diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index 11a610d2f..6ac67d81b 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -183,7 +183,7 @@ t_api(_) -> {ok, 404, ErrResult} = request(get, uri(["authorization", "sources", "http"]), []), ?assertMatch( - #{<<"code">> := <<"NOT_FOUND">>, <<"message">> := <<"Not found http">>}, + #{<<"code">> := <<"NOT_FOUND">>, <<"message">> := <<"Not found: http">>}, jsx:decode(ErrResult) ), From 467010e3d34a72e63f31441665c8256b85e09e3e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 8 Nov 2022 17:05:22 +0100 Subject: [PATCH 151/165] chore: bump emqx_authz app vsn --- apps/emqx_authz/src/emqx_authz.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index 3dd2705d1..6ec14ac3b 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ From 354b94fc2de77b1917f4ab13835e4237401760ee Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 8 Nov 2022 20:59:24 +0100 Subject: [PATCH 152/165] test: ensure all modules start --- apps/emqx/test/emqx_takeover_SUITE.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/test/emqx_takeover_SUITE.erl b/apps/emqx/test/emqx_takeover_SUITE.erl index beb7817af..df17e434a 100644 --- a/apps/emqx/test/emqx_takeover_SUITE.erl +++ b/apps/emqx/test/emqx_takeover_SUITE.erl @@ -33,6 +33,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + emqx_common_test_helpers:boot_modules(all), emqx_channel_SUITE:set_test_listener_confs(), ?check_trace( ?wait_async_action( From 8faa9d8358d1beee302ca9e88e6958597d162b33 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 8 Nov 2022 21:33:29 +0100 Subject: [PATCH 153/165] chore: make nl check script happy --- changes/v5.0.10-en.md | 3 ++- changes/v5.0.10-zh.md | 3 ++- scripts/check-nl-at-eof.sh | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index 59c18566d..b6310a5a2 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -41,4 +41,5 @@ - Align documentation for `/authentication/sources` with what we actually send [9299](https://github.com/emqx/emqx/pull/9299). -- Fix query string parameter 'node' to `/configs` resource being ignored, return 404 if node does not exist [#9310](https://github.com/emqx/emqx/pull/9310/). \ No newline at end of file +- Fix query string parameter 'node' to `/configs` resource being ignored, return 404 if node does not exist [#9310](https://github.com/emqx/emqx/pull/9310/). + diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 62fc633ea..7e057624b 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -40,4 +40,5 @@ - 对齐文档,`/authentication/sources` 接口的文档仅列出已经支持的资源 [9299](https://github.com/emqx/emqx/pull/9299)。 -- 修复 `/configs` API 的 'node' 参数的问题,如果节点不存在,则返回 HTTP 状态码 404 [#9310](https://github.com/emqx/emqx/pull/9310/)。 \ No newline at end of file +- 修复 `/configs` API 的 'node' 参数的问题,如果节点不存在,则返回 HTTP 状态码 404 [#9310](https://github.com/emqx/emqx/pull/9310/)。 + diff --git a/scripts/check-nl-at-eof.sh b/scripts/check-nl-at-eof.sh index 32b774b3b..88f8f9c2e 100755 --- a/scripts/check-nl-at-eof.sh +++ b/scripts/check-nl-at-eof.sh @@ -13,6 +13,9 @@ nl_at_eof() { *.png|*rebar3) return ;; + scripts/erlfmt) + return + ;; esac local lastbyte lastbyte="$(tail -c 1 "$file" 2>&1)" From 6ef7123bb9fb20ec9bbd4bdeedb3e9d790655528 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 7 Nov 2022 10:45:43 +0800 Subject: [PATCH 154/165] chore: bump OTP version --- .../build_and_push_docker_images.yaml | 14 +++++++------- .github/workflows/build_packages.yaml | 18 +++++++++--------- .github/workflows/build_slim_packages.yaml | 6 +++--- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/code_style_check.yaml | 2 +- .github/workflows/elixir_apps_check.yaml | 2 +- .github/workflows/elixir_deps_check.yaml | 2 +- .github/workflows/elixir_release.yml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/run_emqx_app_tests.yaml | 4 ++-- .github/workflows/run_fvt_tests.yaml | 10 +++++----- .github/workflows/run_relup_tests.yaml | 2 +- .github/workflows/run_test_cases.yaml | 9 +++++---- 13 files changed, 38 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 803b50cfc..3372fefc9 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -20,7 +20,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} @@ -112,7 +112,7 @@ jobs: # NOTE: 'otp' and 'elixir' are to configure emqx-builder image # only support latest otp and elixir, not a matrix otp: - - 24.2.1-1 # update to latest + - 24.3.4.2-1 # update to latest elixir: - 1.13.4 # update to latest @@ -164,7 +164,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} RUN_FROM=${{ matrix.os[1] }} EMQX_NAME=${{ steps.meta.outputs.emqx_name }} file: source/${{ matrix.os[2] }} @@ -189,7 +189,7 @@ jobs: os: - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] otp: - - 24.2.1-1 # update to latest + - 24.3.4.2-1 # update to latest elixir: - 1.13.4 # update to latest @@ -232,7 +232,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} RUN_FROM=${{ matrix.os[1] }} EMQX_NAME=${{ steps.meta.outputs.emqx_name }} file: source/${{ matrix.os[2] }} @@ -257,7 +257,7 @@ jobs: - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] # NOTE: only support latest otp version, not a matrix otp: - - 24.2.1-1 # update to latest + - 24.3.4.2-1 # update to latest registry: - 'docker.io' - 'public.ecr.aws' @@ -319,7 +319,7 @@ jobs: - ${{ needs.prepare.outputs.BUILD_PROFILE }} # NOTE: for docker, only support latest otp version, not a matrix otp: - - 24.2.1-1 # update to latest + - 24.3.4.2-1 # update to latest elixir: - 1.13.4 # update to latest registry: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index e4b435a3e..36ec90283 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -23,7 +23,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04 outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} @@ -140,7 +140,7 @@ jobs: profile: - ${{ needs.prepare.outputs.BUILD_PROFILE }} otp: - - 24.2.1-1 + - 24.3.4.2-1 os: - macos-11 runs-on: ${{ matrix.os }} @@ -175,7 +175,7 @@ jobs: needs: prepare runs-on: ${{ matrix.build_machine }} container: - image: "ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" + image: "ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" strategy: fail-fast: false @@ -183,7 +183,7 @@ jobs: profile: - ${{ needs.prepare.outputs.BUILD_PROFILE }} otp: - - 24.2.1-1 # we test with OTP 23, but only build package on OTP 24 versions + - 24.3.4.2-1 # we test with OTP 23, but only build package on OTP 24 versions elixir: - 1.13.4 # used to split elixir packages into a separate job, since the @@ -232,14 +232,14 @@ jobs: profile: emqx-enterprise include: - profile: emqx - otp: 24.2.1-1 + otp: 24.3.4.2-1 elixir: 1.13.4 build_elixir: with_elixir arch: amd64 os: ubuntu20.04 build_machine: ubuntu-20.04 - profile: emqx - otp: 24.2.1-1 + otp: 24.3.4.2-1 elixir: 1.13.4 build_elixir: with_elixir arch: amd64 @@ -290,7 +290,7 @@ jobs: --pkgtype "${PKGTYPE}" \ --arch "${ARCH}" \ --elixir "${IsElixir}" \ - --builder "ghcr.io/emqx/emqx-builder/5.0-17:${ELIXIR}-${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/5.0-18:${ELIXIR}-${OTP}-${SYSTEM}" done - uses: actions/upload-artifact@v3 with: @@ -307,7 +307,7 @@ jobs: profile: - ${{ needs.prepare.outputs.BUILD_PROFILE }} otp: - - 24.2.1-1 + - 24.3.4.2-1 include: - profile: emqx otp: windows # otp version on windows is rather fixed @@ -320,7 +320,7 @@ jobs: run: sudo apt-get update && sudo apt install -y dos2unix - name: get packages run: | - DEFAULT_BEAM_PLATFORM='otp24.2.1-1' + DEFAULT_BEAM_PLATFORM='otp24.3.4.2-1' set -e -u cd packages/${{ matrix.profile }} # Make a copy of the default OTP version package to a file without OTP version infix diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 8f2e43f45..0e3e3d036 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -32,14 +32,14 @@ jobs: - emqx - emqx-enterprise otp: - - 24.2.1-1 + - 24.3.4.2-1 elixir: - 1.13.4 os: - ubuntu20.04 - el8 - container: "ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" steps: - uses: AutoModality/action-clean@v1 @@ -132,7 +132,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.2.1-1 + - 24.3.4.2-1 os: - macos-11 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index fdf8903d4..c5c509f0c 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request, push] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml index 1417960ed..bc15d696f 100644 --- a/.github/workflows/code_style_check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: code_style_check: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index 2ec54ce63..8dc9e54cd 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -8,7 +8,7 @@ jobs: elixir_apps_check: runs-on: ubuntu-latest # just use the latest builder - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" strategy: fail-fast: false diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index 2cecade04..210eda570 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -7,7 +7,7 @@ on: [pull_request, push] jobs: elixir_deps_check: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04 steps: - name: Checkout diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index 36ba5496d..cef1095a2 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -12,7 +12,7 @@ on: jobs: elixir_release_build: runs-on: ubuntu-latest - container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04 steps: - name: Checkout diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5569aa300..3e7dd4ada 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -32,7 +32,7 @@ jobs: esac aws s3 cp --recursive s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} packages cd packages - DEFAULT_BEAM_PLATFORM='otp24.2.1-1' + DEFAULT_BEAM_PLATFORM='otp24.3.4.2-1' # all packages including full-name and default-name are uploaded to s3 # but we only upload default-name packages (and elixir) as github artifacts # so we rename (overwrite) non-default packages before uploading diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index c05fa7a3a..5b68e3dda 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: otp: - - 24.2.1-1 + - 24.3.4.2-1 # no need to use more than 1 version of Elixir, since tests # run using only Erlang code. This is needed just to specify # the base image. @@ -24,7 +24,7 @@ jobs: - amd64 runs-on: aws-amd64 - container: "ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" defaults: run: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 6a6466442..0464b5e50 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -16,7 +16,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-alpine3.15.1 + container: ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-alpine3.15.1 steps: - uses: actions/checkout@v3 @@ -49,7 +49,7 @@ jobs: os: - ["alpine3.15.1", "alpine:3.15.1"] otp: - - 24.2.1-1 + - 24.3.4.2-1 elixir: - 1.13.4 arch: @@ -68,7 +68,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} EMQX_RUNNER: ${{ matrix.os[1] }} run: | make ${{ matrix.profile }}-docker @@ -120,7 +120,7 @@ jobs: os: - ["debian11", "debian:11-slim"] otp: - - 24.2.1-1 + - 24.3.4.2-1 elixir: - 1.13.4 arch: @@ -141,7 +141,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} EMQX_RUNNER: ${{ matrix.os[1] }} run: | make ${{ matrix.profile }}-docker diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 6f88c78e4..ea0dff9e9 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -16,7 +16,7 @@ on: jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" outputs: CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 53939aaf3..03030908c 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -17,7 +17,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" outputs: fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }} docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }} @@ -60,7 +60,7 @@ jobs: defaults: run: shell: bash - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" steps: - uses: AutoModality/action-clean@v1 @@ -121,6 +121,7 @@ jobs: PGSQL_TAG: 13 REDIS_TAG: 6 run: | + rm _build/default/lib/rocksdb/_build/cmake/CMakeCache.txt ./scripts/ct/run.sh --app ${{ matrix.app_name }} - uses: actions/upload-artifact@v3 with: @@ -143,7 +144,7 @@ jobs: - emqx-enterprise runs-on: aws-amd64 - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" defaults: run: shell: bash @@ -200,7 +201,7 @@ jobs: - ct - ct_docker runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 From d90b3760b1563a825b42ff539d10da5fbf5c226b Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 8 Nov 2022 15:00:51 +0100 Subject: [PATCH 155/165] ci(ct/run.sh): use host uid and gid in erlang container for ct --- .ci/docker-compose-file/docker-compose.yaml | 2 ++ scripts/ct/run.sh | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 2612eb8d8..8e723f117 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -18,8 +18,10 @@ services: - emqx_bridge volumes: - ../..:/emqx + - ./.cache:/.cache working_dir: /emqx tty: true + user: "${UID_GID}" networks: emqx_bridge: diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 879efc434..00e2278cf 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -90,7 +90,7 @@ for file in "${FILES[@]}"; do done # shellcheck disable=2086 # no quotes for F_OPTIONS -docker-compose $F_OPTIONS up -d --build +UID_GID="$(id -u):$(id -g)" docker-compose $F_OPTIONS up -d --build # /emqx is where the source dir is mounted to the Erlang container # in .ci/docker-compose-file/docker-compose.yaml @@ -98,7 +98,6 @@ TTY='' if [[ -t 1 ]]; then TTY='-t' fi -docker exec -i $TTY "$ERLANG_CONTAINER" bash -c 'git config --global --add safe.directory /emqx' if [ "$CONSOLE" = 'yes' ]; then docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "make run" @@ -107,6 +106,6 @@ else docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "make ${WHICH_APP}-ct" RESULT=$? # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS down + UID_GID="$(id -u):$(id -g)" docker-compose $F_OPTIONS down exit $RESULT fi From 9b999f070d70e388025e76ae46c4f581742a2caf Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 9 Nov 2022 11:03:01 +0800 Subject: [PATCH 156/165] chore: make the ./cache to be writeable --- scripts/ct/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 00e2278cf..94b57d536 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -90,6 +90,7 @@ for file in "${FILES[@]}"; do done # shellcheck disable=2086 # no quotes for F_OPTIONS +mkdir -p ./.cache && chmod a+rwx ./.cache UID_GID="$(id -u):$(id -g)" docker-compose $F_OPTIONS up -d --build # /emqx is where the source dir is mounted to the Erlang container From 7bbd6353aa7c62378355ddb66cf9a0804c5f22e0 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 9 Nov 2022 09:30:32 +0100 Subject: [PATCH 157/165] chore: fix /.cache and /.erlang.cookie in erlang container --- .ci/docker-compose-file/docker-compose.yaml | 1 - scripts/ct/run.sh | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 8e723f117..3d99d3969 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -18,7 +18,6 @@ services: - emqx_bridge volumes: - ../..:/emqx - - ./.cache:/.cache working_dir: /emqx tty: true user: "${UID_GID}" diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 94b57d536..f6956a51b 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -90,8 +90,8 @@ for file in "${FILES[@]}"; do done # shellcheck disable=2086 # no quotes for F_OPTIONS -mkdir -p ./.cache && chmod a+rwx ./.cache -UID_GID="$(id -u):$(id -g)" docker-compose $F_OPTIONS up -d --build +UID_GID="$(id -u):$(id -g)" +UID_GID=$UID_GID docker-compose $F_OPTIONS up -d --build # /emqx is where the source dir is mounted to the Erlang container # in .ci/docker-compose-file/docker-compose.yaml @@ -100,6 +100,8 @@ if [[ -t 1 ]]; then TTY='-t' fi +docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir /.cache && chown $UID_GID /.cache" +docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "touch /.erlang.cookie && chown $UID_GID /.erlang.cookie" if [ "$CONSOLE" = 'yes' ]; then docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "make run" else @@ -107,6 +109,6 @@ else docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "make ${WHICH_APP}-ct" RESULT=$? # shellcheck disable=2086 # no quotes for F_OPTIONS - UID_GID="$(id -u):$(id -g)" docker-compose $F_OPTIONS down + UID_GID=$UID_GID docker-compose $F_OPTIONS down exit $RESULT fi From c1255188dbdc988e38abf38923b57698be66000c Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 9 Nov 2022 10:08:59 +0100 Subject: [PATCH 158/165] chore: manually initialize .erlang.cookie for ct_docker --- scripts/ct/run.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index f6956a51b..aa6f55022 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -89,9 +89,13 @@ for file in "${FILES[@]}"; do F_OPTIONS="$F_OPTIONS -f $file" done +# Passing $UID to docker-compose to be used in erlang container +# as owner of the main process to avoid git repo permissions issue. +# Permissions issue happens because we are mounting local filesystem +# where files are owned by $UID to docker container where it's using +# root (UID=0) by default, and git is not happy about it. # shellcheck disable=2086 # no quotes for F_OPTIONS -UID_GID="$(id -u):$(id -g)" -UID_GID=$UID_GID docker-compose $F_OPTIONS up -d --build +UID_GID="$UID:$UID" docker-compose $F_OPTIONS up -d --build # /emqx is where the source dir is mounted to the Erlang container # in .ci/docker-compose-file/docker-compose.yaml @@ -100,8 +104,10 @@ if [[ -t 1 ]]; then TTY='-t' fi -docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir /.cache && chown $UID_GID /.cache" -docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "touch /.erlang.cookie && chown $UID_GID /.erlang.cookie" +# rebar and hex cache directory need to be writable by $UID +docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir /.cache && chown $UID:$UID /.cache" +# need to initialize .erlang.cookie manually here because / is not writable by $UID +docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $UID:$UID /.erlang.cookie && chmod 0400 /.erlang.cookie" if [ "$CONSOLE" = 'yes' ]; then docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "make run" else @@ -109,6 +115,6 @@ else docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "make ${WHICH_APP}-ct" RESULT=$? # shellcheck disable=2086 # no quotes for F_OPTIONS - UID_GID=$UID_GID docker-compose $F_OPTIONS down + UID_GID="$UID:$UID" docker-compose $F_OPTIONS down exit $RESULT fi From a03271d6b20c711440ba8d53994e7a37ff110996 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 9 Nov 2022 11:26:05 +0100 Subject: [PATCH 159/165] docs: add missing change logs --- changes/v5.0.10-en.md | 8 +++++++- changes/v5.0.10-zh.md | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index b6310a5a2..2545ac1bf 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -16,7 +16,11 @@ - Now it is possible to opt out VM internal metrics in prometheus stats [#9222](https://github.com/emqx/emqx/pull/9222). When system load is high, reporting too much metrics data may cause the prometheus stats API timeout. -- Improve security when converting types such as `binary` `lists` to `atom` types [#9279](https://github.com/emqx/emqx/pull/9279). +- Improve security when converting types such as `binary` `lists` to `atom` types [#9279](https://github.com/emqx/emqx/pull/9279), [#9286](https://github.com/emqx/emqx/pull/9286). + +- Add `/trace/:name/log_detail` HTTP API to return trace file's size and mtime [#9152](https://github.com/emqx/emqx/pull/9152). + +- Add `/status` HTTP API endpoint to api documentation [#9230](https://github.com/emqx/emqx/pull/9230). ## Bug fixes @@ -43,3 +47,5 @@ - Fix query string parameter 'node' to `/configs` resource being ignored, return 404 if node does not exist [#9310](https://github.com/emqx/emqx/pull/9310/). +- Avoid re-dispatching shared-subscription session messages when a session is kicked or taken-over (to a new session) [#9123](https://github.com/emqx/emqx/pull/9123). + diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 7e057624b..31726e858 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -15,7 +15,11 @@ - 可通过配置关闭 prometheus 中的部分内部指标,如果遇到机器负载过高 prometheus 接口返回超时可考虑关闭部分不关心指标,以提高响应速度 [#9222](https://github.com/emqx/emqx/pull/9222)。 -- 提升 `binary` 、`list` 等类型转换为 `atom` 类型时的安全性 [#9279](https://github.com/emqx/emqx/pull/9279)。 +- 提升 `binary` 、`list` 等类型转换为 `atom` 类型时的安全性 [#9279](https://github.com/emqx/emqx/pull/9279),[#9286](https://github.com/emqx/emqx/pull/9286)。 + +- 增加了 `/trace/:name/log_detail` HTTP API 用于返回 trace 文件的大小和修改日期等信息 [#9152](https://github.com/emqx/emqx/pull/9152)。 + +- HTTP API 文档中增加 `/status` 端点的描述 [#9230](https://github.com/emqx/emqx/pull/9230)。 ## Bug fixes @@ -42,3 +46,4 @@ - 修复 `/configs` API 的 'node' 参数的问题,如果节点不存在,则返回 HTTP 状态码 404 [#9310](https://github.com/emqx/emqx/pull/9310/)。 +- 共享订阅的消息在会话被踢出或者迁移时,不向其他订阅组成员进行转发 [#9123](https://github.com/emqx/emqx/pull/9123)。 From 715c2d856d17dcfcb41763a7730294db62a15e54 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 9 Nov 2022 11:28:17 +0100 Subject: [PATCH 160/165] chore: upgrade dashboard version to v1.1.1 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7644e1e2a..ea617a248 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.1.0 +export EMQX_DASHBOARD_VERSION ?= v1.1.1 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.0 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 From 493b91782ddd491edabf60c0d28e4867f9f6d061 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 9 Nov 2022 11:54:24 +0100 Subject: [PATCH 161/165] ci: fix upload artifacts path --- .github/workflows/build_packages.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index e4b435a3e..eee750560 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -130,7 +130,7 @@ jobs: - uses: actions/upload-artifact@v3 with: name: ${{ matrix.profile }}-windows - path: source/_packages/${{ matrix.profile }}/. + path: source/_packages/${{ matrix.profile }}/ mac: needs: prepare @@ -169,7 +169,7 @@ jobs: - uses: actions/upload-artifact@v3 with: name: ${{ matrix.profile }}-${{ matrix.otp }} - path: _packages/${{ matrix.profile }}/. + path: _packages/${{ matrix.profile }}/ linux: needs: prepare From 7f2fb0ea5791c7772b66f3788e69a09381cc4d80 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 9 Nov 2022 13:02:50 +0100 Subject: [PATCH 162/165] ci: fix artifacts upload for linux build --- .github/workflows/build_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index eee750560..98bbaa73b 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -295,7 +295,7 @@ jobs: - uses: actions/upload-artifact@v3 with: name: ${{ matrix.profile }}-${{ matrix.otp }} - path: source/_packages/${{ matrix.profile }}/. + path: source/_packages/${{ matrix.profile }}/ publish_artifacts: runs-on: ubuntu-20.04 From c186bc2c55c24cacf2742bd7e80dfdd5413246ff Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 9 Nov 2022 13:12:21 +0100 Subject: [PATCH 163/165] docs: add v5.0.11 change logs file --- changes/v5.0.11-en.md | 6 ++++++ changes/v5.0.11-zh.md | 5 +++++ 2 files changed, 11 insertions(+) create mode 100644 changes/v5.0.11-en.md create mode 100644 changes/v5.0.11-zh.md diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md new file mode 100644 index 000000000..9fbc2225f --- /dev/null +++ b/changes/v5.0.11-en.md @@ -0,0 +1,6 @@ +# v5.0.11 + +## Enhancements + +## Bug fixes + diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md new file mode 100644 index 000000000..cea0f10fb --- /dev/null +++ b/changes/v5.0.11-zh.md @@ -0,0 +1,5 @@ +# v5.0.11 + +## 增强 + +## 修复 From 7e8d31b219ce49755d6ffeaa9cf54bfaeb9530fb Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 9 Nov 2022 17:24:56 +0100 Subject: [PATCH 164/165] docs: update v5.0.10 changelog for #9293 --- changes/v5.0.10-en.md | 2 ++ changes/v5.0.10-zh.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md index b6310a5a2..60ccb1054 100644 --- a/changes/v5.0.10-en.md +++ b/changes/v5.0.10-en.md @@ -18,6 +18,8 @@ - Improve security when converting types such as `binary` `lists` to `atom` types [#9279](https://github.com/emqx/emqx/pull/9279). +- Binary packages for all platforms are now built on Erlang/OTP version 24.3.4.2 [#9293](https://github.com/emqx/emqx/pull/9293). + ## Bug fixes - Fix error log message when `mechanism` is missing in authentication config [#8924](https://github.com/emqx/emqx/pull/8924). diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md index 7e057624b..188008326 100644 --- a/changes/v5.0.10-zh.md +++ b/changes/v5.0.10-zh.md @@ -17,6 +17,8 @@ - 提升 `binary` 、`list` 等类型转换为 `atom` 类型时的安全性 [#9279](https://github.com/emqx/emqx/pull/9279)。 +- 为所有平台的二进制包升级了 Erlang/OTP 到 24.3.4.2 [#9293](https://github.com/emqx/emqx/pull/9293). + ## Bug fixes - 优化认认证配置中 `mechanism` 字段缺失情况下的错误日志 [#8924](https://github.com/emqx/emqx/pull/8924)。 From 155d4a98184b189a45c22db37198f5e54d9be430 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 9 Nov 2022 19:11:48 +0100 Subject: [PATCH 165/165] ci: fix yaml syntax --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3e7dd4ada..05dcb62e3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -87,7 +87,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - ref: ${{ github.ref } + ref: ${{ github.ref }} - uses: emqx/push-helm-action@v1 if: startsWith(github.ref_name, 'v') with: