From b4c264329145c3779afe80d0533ada2f5e80052f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 22 Oct 2021 09:21:34 +0800 Subject: [PATCH 001/104] fix(api-clients): escape the searching string --- apps/emqx_management/src/emqx_mgmt_api_clients.erl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 2fe6a5ccb..1ddd87a3d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -334,7 +334,7 @@ query({Qs, Fuzzy}, Start, Limit) -> match_fun(Ms, Fuzzy) -> MsC = ets:match_spec_compile(Ms), REFuzzy = lists:map(fun({K, like, S}) -> - {ok, RE} = re:compile(S), + {ok, RE} = re:compile(escape(S)), {K, like, RE} end, Fuzzy), fun(Rows) -> @@ -347,6 +347,9 @@ match_fun(Ms, Fuzzy) -> end end. +escape(B) when is_binary(B) -> + re:replace(B, <<"\\\\">>, <<"\\\\\\\\">>, [{return, binary}, global]). + run_fuzzy_match(_, []) -> true; run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) -> @@ -450,4 +453,9 @@ params2qs_test() -> [{{'$1', #{}, '_'}, [], ['$_']}] = qs2ms([]). +escape_test() -> + Str = <<"\\n">>, + {ok, Re} = re:compile(escape(Str)), + {match, _} = re:run(<<"\\name">>, Re). + -endif. From 791caba2ede8710bb9600e9b66b5bbb8153d1123 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Fri, 29 Oct 2021 09:55:19 +0200 Subject: [PATCH 002/104] fix(broker): Fix out-of-order message delivery in a cluster Fixes: #4658 --- etc/emqx.conf | 2 +- priv/emqx.schema | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 6043dc361..f24d7ada2 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -344,7 +344,7 @@ rpc.port_discovery = stateless ## ## Value: Interger [0-256] ## Default = 1 -#rpc.tcp_client_num = 1 +#rpc.tcp_client_num = 0 ## RCP Client connect timeout. ## diff --git a/priv/emqx.schema b/priv/emqx.schema index 86a2a2892..4b33cf65c 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -376,7 +376,7 @@ end}. {translation, "gen_rpc.tcp_client_num", fun(Conf) -> case cuttlefish:conf_get("rpc.tcp_client_num", Conf) of - 0 -> 1; %% keep allowing 0 for backward compatibility + 0 -> max(1, erlang:system_info(schedulers) div 2); V -> V end end}. diff --git a/rebar.config b/rebar.config index f56e8e2c2..bd39f9816 100644 --- a/rebar.config +++ b/rebar.config @@ -44,7 +44,7 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.3"}}} - , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} + , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.6.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} From f8acb31f89baff71424a3ac878f09b675ffb78a2 Mon Sep 17 00:00:00 2001 From: lafirest Date: Wed, 27 Oct 2021 11:43:05 +0800 Subject: [PATCH 003/104] feat: add slow topics statistics plugin --- .../etc/emqx_st_statistics.conf | 38 ++ .../priv/emqx_st_statistics.schema | 58 +++ apps/emqx_st_statistics/rebar.config | 23 ++ .../src/emqx_st_statistics.app.src | 12 + .../src/emqx_st_statistics.appup.src | 9 + .../src/emqx_st_statistics.erl | 331 ++++++++++++++++++ .../src/emqx_st_statistics_app.erl | 33 ++ .../src/emqx_st_statistics_sup.erl | 35 ++ .../test/emqx_st_statistics_SUITE.erl | 110 ++++++ rebar.config.erl | 1 + scripts/apps-version-check.sh | 90 +++-- src/emqx_session.erl | 9 +- 12 files changed, 714 insertions(+), 35 deletions(-) create mode 100644 apps/emqx_st_statistics/etc/emqx_st_statistics.conf create mode 100644 apps/emqx_st_statistics/priv/emqx_st_statistics.schema create mode 100644 apps/emqx_st_statistics/rebar.config create mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics.app.src create mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics.appup.src create mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics.erl create mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics_app.erl create mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl create mode 100644 apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl diff --git a/apps/emqx_st_statistics/etc/emqx_st_statistics.conf b/apps/emqx_st_statistics/etc/emqx_st_statistics.conf new file mode 100644 index 000000000..88ca0c37f --- /dev/null +++ b/apps/emqx_st_statistics/etc/emqx_st_statistics.conf @@ -0,0 +1,38 @@ +##-------------------------------------------------------------------- +## EMQ X Slow Topics Statistics +##-------------------------------------------------------------------- + +## Threshold time of slow topics statistics +## +## Default: 10 seconds +st_statistics.threshold_time = 10S + +## Time window of slow topics statistics +## +## Value: 5 minutes +st_statistics.time_window = 5M + +## Maximum of slow topics log, log will clear when enter new time window +## +## Value: 500 +st_statistics.max_log_num = 500 + +## Top-K record for slow topics, update from logs +## +## Value: 500 +st_statistics.top_k_num = 500 + +## Topic of notification +## +## Defaut: $slow_topics +st_statistics.notice_topic = $slow_topics + +## QoS of notification message in notice topic +## +## Defaut: 0 +st_statistics.notice_qos = 0 + +## Maximum information number in one notification +## +## Default: 500 +st_statistics.notice_batch_size = 500 diff --git a/apps/emqx_st_statistics/priv/emqx_st_statistics.schema b/apps/emqx_st_statistics/priv/emqx_st_statistics.schema new file mode 100644 index 000000000..da9a2f810 --- /dev/null +++ b/apps/emqx_st_statistics/priv/emqx_st_statistics.schema @@ -0,0 +1,58 @@ +%%-*- mode: erlang -*- +%% st_statistics config mapping + +%% Threshold time of slow topics statistics +%% {$configurable} +{mapping, "st_statistics.threshold_time", "emqx_st_statistics.threshold_time", + [ + {default, "10S"}, + {datatype, [integer, {duration, ms}]} + ]}. + +%% Time window of slow topics statistics +%% {$configurable} +{mapping, "st_statistics.time_window", "emqx_st_statistics.time_window", + [ + {default, "5M"}, + {datatype, [integer, {duration, ms}]} + ]}. + +%% Maximum of slow topics log +%% {$configurable} +{mapping, "st_statistics.max_log_num", "emqx_st_statistics.max_log_num", + [ + {default, 500}, + {datatype, integer} + ]}. + +%% Top-K record for slow topics, update from logs +%% {$configurable} +{mapping, "st_statistics.top_k_num", "emqx_st_statistics.top_k_num", + [ + {default, 500}, + {datatype, integer} + ]}. + +%% Topic of notification +%% {$configurable} +{mapping, "st_statistics.notice_topic", "emqx_st_statistics.notice_topic", + [ + {default, <<"slow_topics">>}, + {datatype, string} + ]}. + +%% QoS of notification message in notice topic +%% {$configurable} +{mapping, "st_statistics.notice_qos", "emqx_st_statistics.notice_qos", + [ + {default, 0}, + {datatype, integer} + ]}. + +%% Maximum entities per notification message +%% {$configurable} +{mapping, "st_statistics.notice_batch_size", "emqx_st_statistics.notice_batch_size", + [ + {default, 500}, + {datatype, integer} + ]}. diff --git a/apps/emqx_st_statistics/rebar.config b/apps/emqx_st_statistics/rebar.config new file mode 100644 index 000000000..6433d92d6 --- /dev/null +++ b/apps/emqx_st_statistics/rebar.config @@ -0,0 +1,23 @@ +{deps, []}. + +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard + ]}. + +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions]}. +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +{profiles, + [{test, + [{deps, + [{emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}, + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3"}}}]} + ]} + ]}. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.app.src b/apps/emqx_st_statistics/src/emqx_st_statistics.app.src new file mode 100644 index 000000000..b1eb1612a --- /dev/null +++ b/apps/emqx_st_statistics/src/emqx_st_statistics.app.src @@ -0,0 +1,12 @@ +{application, emqx_st_statistics, + [{description, "EMQ X Slow Topics Statistics"}, + {vsn, "1.0.0"}, % strict semver, bump manually! + {modules, []}, + {registered, [emqx_st_statistics_sup]}, + {applications, [kernel,stdlib]}, + {mod, {emqx_st_statistics_app,[]}}, + {env, []}, + {licenses, ["Apache-2.0"]}, + {maintainers, ["EMQ X Team "]}, + {links, []} + ]}. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.appup.src b/apps/emqx_st_statistics/src/emqx_st_statistics.appup.src new file mode 100644 index 000000000..dcf0d8cdd --- /dev/null +++ b/apps/emqx_st_statistics/src/emqx_st_statistics.appup.src @@ -0,0 +1,9 @@ +%% -*-: erlang -*- +{VSN, + [ + {<<".*">>, []} + ], + [ + {<<".*">>, []} + ] +}. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.erl b/apps/emqx_st_statistics/src/emqx_st_statistics.erl new file mode 100644 index 000000000..8a6c1535f --- /dev/null +++ b/apps/emqx_st_statistics/src/emqx_st_statistics.erl @@ -0,0 +1,331 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_st_statistics). + +-behaviour(gen_server). + +-include_lib("include/emqx.hrl"). +-include_lib("include/logger.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). + +-logger_header("[SLOW TOPICS]"). + +-export([ start_link/1, on_publish_done/3, enable/0 + , disable/0 + ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-compile(nowarn_unused_type). + +-type state() :: #{ config := proplist:proplist() + , index := index_map() + , begin_time := pos_integer() + , counter := counters:counter_ref() + , enable := boolean() + }. + +-type log() :: #{ topic := emqx_types:topic() + , times := pos_integer() + , average := float() + }. + +-type window_log() :: #{ begin_time := pos_integer() + , logs := [log()] + }. + +-record(slow_log, { topic :: emqx_types:topic() + , times :: non_neg_integer() + , elapsed :: non_neg_integer() + }). + +-record(top_k, { key :: any() + , average :: float()}). + +-type message() :: #message{}. + +-import(proplists, [get_value/2]). + +-define(LOG_TAB, emqx_st_statistics_log). +-define(TOPK_TAB, emqx_st_statistics_topk). +-define(NOW, erlang:system_time(millisecond)). +-define(TOP_KEY(Times, Topic), {Times, Topic}). +-define(QUOTA_IDX, 1). + +-type top_key() :: ?TOP_KEY(pos_integer(), emqx_types:topic()). +-type index_map() :: #{emqx_types:topic() => pos_integer()}. + +%% erlang term order +%% number < atom < reference < fun < port < pid < tuple < list < bit string + +%% ets ordered_set is ascending by term order + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +%% @doc Start the st_statistics +-spec(start_link(Env :: list()) -> emqx_types:startlink_ret()). +start_link(Env) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). + +-spec on_publish_done(message(), pos_integer(), counters:counters_ref()) -> ok. +on_publish_done(#message{timestamp = Timestamp} = Msg, Threshold, Counter) -> + case ?NOW - Timestamp of + Elapsed when Elapsed > Threshold -> + case get_log_quota(Counter) of + true -> + update_log(Msg, Elapsed); + _ -> + ok + end; + _ -> + ok + end. + +enable() -> + gen_server:call(?MODULE, {enable, true}). + +disable() -> + gen_server:call(?MODULE, {enable, false}). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([Env]) -> + erlang:process_flag(trap_exit, true), + init_log_tab(Env), + init_topk_tab(Env), + notification_tick(Env), + Counter = counters:new(1, [write_concurrency]), + set_log_quota(Env, Counter), + Threshold = get_value(threshold_time, Env), + load(Threshold, Counter), + {ok, #{config => Env, + index => #{}, + begin_time => ?NOW, + counter => Counter, + enable => true}}. + +handle_call({enable, Enable}, _From, + #{config := Cfg, counter := Counter, enable := IsEnable} = State) -> + State2 = case Enable of + IsEnable -> + State; + true -> + Threshold = get_value(threshold_time, Cfg), + load(Threshold, Counter), + State#{enable := true}; + _ -> + unload(), + State#{enable := false} + end, + {reply, ok, State2}; + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info(notification_tick, #{config := Cfg} = State) -> + notification_tick(Cfg), + Index2 = do_notification(State), + {noreply, State#{index := Index2, + begin_time := ?NOW}}; + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _) -> + unload(), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +notification_tick(Env) -> + TimeWindow = get_value(time_window, Env), + erlang:send_after(TimeWindow, self(), ?FUNCTION_NAME). + +init_log_tab(_) -> + ?LOG_TAB = ets:new(?LOG_TAB, [ set, public, named_table + , {keypos, #slow_log.topic}, {write_concurrency, true} + , {read_concurrency, true} + ]). + +init_topk_tab(_) -> + ?TOPK_TAB = ets:new(?TOPK_TAB, [ ordered_set, protected, named_table + , {keypos, #top_k.key}, {write_concurrency, true} + , {read_concurrency, false} + ]). + +-spec get_log_quota(counter:counter_ref()) -> boolean(). +get_log_quota(Counter) -> + case counters:get(Counter, ?QUOTA_IDX) of + Quota when Quota > 0 -> + counters:sub(Counter, ?QUOTA_IDX, 1), + true; + _ -> + false + end. + +-spec set_log_quota(proplists:proplist(), counter:counter_ref()) -> ok. +set_log_quota(Cfg, Counter) -> + MaxLogNum = get_value(max_log_num, Cfg), + counters:put(Counter, ?QUOTA_IDX, MaxLogNum). + +-spec update_log(message(), non_neg_integer()) -> ok. +update_log(#message{topic = Topic}, Elapsed) -> + _ = ets:update_counter(?LOG_TAB, + Topic, + [{#slow_log.times, 1}, {#slow_log.elapsed, Elapsed}], + #slow_log{topic = Topic, + times = 1, + elapsed = Elapsed}), + ok. + +-spec do_notification(state()) -> index_map(). +do_notification(#{begin_time := BeginTime, + config := Cfg, + index := IndexMap, + counter := Counter}) -> + Logs = ets:tab2list(?LOG_TAB), + ets:delete_all_objects(?LOG_TAB), + start_publish(Logs, BeginTime, Cfg), + set_log_quota(Cfg, Counter), + MaxRecord = get_value(top_k_num, Cfg), + Size = ets:info(?TOPK_TAB, size), + update_top_k(Logs, erlang:max(0, MaxRecord - Size), IndexMap). + +-spec update_top_k(list(#slow_log{}), non_neg_integer(), index_map()) -> index_map(). +update_top_k([#slow_log{topic = Topic, + times = NewTimes, + elapsed = Elapsed} = Log | T], + Left, + IndexMap) -> + case maps:get(Topic, IndexMap, 0) of + 0 -> + try_insert_new(Log, Left, T, IndexMap); + Times -> + [#top_k{key = Key, average = Average}] = ets:lookup(?TOPK_TAB, ?TOP_KEY(Times, Topic)), + Times2 = Times + NewTimes, + Total = Times * Average + Elapsed, + Average2 = Total / Times2, + ets:delete(?TOPK_TAB, Key), + ets:insert(?TOPK_TAB, #top_k{key = ?TOP_KEY(Times2, Topic), average = Average2}), + update_top_k(T, Left, IndexMap#{Topic := Times2}) + end; + +update_top_k([], _, IndexMap) -> + IndexMap. + +-spec try_insert_new(#slow_log{}, + non_neg_integer(), list(#slow_log{}), index_map()) -> index_map(). +try_insert_new(#slow_log{topic = Topic, + times = Times, + elapsed = Elapsed}, Left, Logs, IndexMap) when Left > 0 -> + Average = Elapsed / Times, + ets:insert_new(?TOPK_TAB, #top_k{key = ?TOP_KEY(Times, Topic), average = Average}), + update_top_k(Logs, Left - 1, IndexMap#{Topic => Times}); + +try_insert_new(#slow_log{topic = Topic, + times = Times, + elapsed = Elapsed}, Left, Logs, IndexMap) -> + ?TOP_KEY(MinTimes, MinTopic) = MinKey = ets:first(?TOPK_TAB), + case MinTimes > Times of + true -> + update_top_k(Logs, Left, IndexMap); + _ -> + Average = Elapsed / Times, + ets:delete(?TOPK_TAB, MinKey), + ets:insert_new(?TOPK_TAB, #top_k{key = ?TOP_KEY(Times, Topic), average = Average}), + update_top_k(Logs, + Left - 1, + maps:put(Topic, Times, maps:remove(MinTopic, IndexMap))) + end. + +start_publish(Logs, BeginTime, Cfg) -> + emqx_pool:async_submit({fun do_publish/3, [Logs, BeginTime, Cfg]}). + +do_publish([], _, _) -> + ok; + +do_publish(Logs, BeginTime, Cfg) -> + BatchSize = get_value(notice_batch_size, Cfg), + do_publish(Logs, BatchSize, BeginTime, Cfg, []). + +do_publish([Log | T], Size, BeginTime, Cfg, Cache) when Size > 0 -> + Cache2 = [convert_to_notice(Log) | Cache], + do_publish(T, Size - 1, BeginTime, Cfg, Cache2); + +do_publish(Logs, Size, BeginTime, Cfg, Cache) when Size =:= 0 -> + publish(BeginTime, Cfg, Cache), + do_publish(Logs, BeginTime, Cfg); + +do_publish([], _, BeginTime, Cfg, Cache) -> + publish(BeginTime, Cfg, Cache), + ok. + +convert_to_notice(#slow_log{topic = Topic, + times = Times, + elapsed = Elapsed}) -> + #{topic => Topic, + times => Times, + average => Elapsed / Times}. + +publish(BeginTime, Cfg, Notices) -> + WindowLog = #{begin_time => BeginTime, + logs => Notices}, + Payload = emqx_json:encode(WindowLog), + _ = emqx:publish(#message{ id = emqx_guid:gen() + , qos = get_value(notice_qos, Cfg) + , from = ?MODULE + , topic = get_topic(Cfg) + , payload = Payload + , timestamp = ?NOW + }), + ok. + +load(Threshold, Counter) -> + _ = emqx:hook('message.publish_done', fun ?MODULE:on_publish_done/3, [Threshold, Counter]), + ok. + +unload() -> + emqx:unhook('message.publish_done', fun ?MODULE:on_publish_done/3). + +get_topic(Cfg) -> + case get_value(notice_topic, Cfg) of + Topic when is_binary(Topic) -> + Topic; + Topic -> + erlang:list_to_binary(Topic) + end. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl b/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl new file mode 100644 index 000000000..e5b62a00e --- /dev/null +++ b/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_st_statistics_app). + +-behaviour(application). + +-emqx_plugin(?MODULE). + +-export([ start/2 + , stop/1 + ]). + +start(_Type, _Args) -> + Env = application:get_all_env(emqx_st_statistics), + {ok, Sup} = emqx_st_statistics_sup:start_link(Env), + {ok, Sup}. + +stop(_State) -> + ok. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl b/apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl new file mode 100644 index 000000000..cbff854ef --- /dev/null +++ b/apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl @@ -0,0 +1,35 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_st_statistics_sup). + +-behaviour(supervisor). + +-export([start_link/1]). + +-export([init/1]). + +start_link(Env) -> + supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]). + +init([Env]) -> + {ok, {{one_for_one, 10, 3600}, + [#{id => st_statistics, + start => {emqx_st_statistics, start_link, [Env]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_st_statistics]}]}}. diff --git a/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl b/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl new file mode 100644 index 000000000..c08c8d986 --- /dev/null +++ b/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl @@ -0,0 +1,110 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_st_statistics_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("include/emqx.hrl"). + +%-define(LOGT(Format, Args), ct:pal(Format, Args)). + +-define(LOG_TAB, emqx_st_statistics_log). +-define(TOPK_TAB, emqx_st_statistics_topk). +-define(NOW, erlang:system_time(millisecond)). + +all() -> emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_st_statistics], fun set_special_cfg/1), + Config. + +set_special_cfg(_) -> + application:set_env([{emqx_st_statistics, base_conf()}]), + ok. + +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx_st_statistics]), + Config. + +%%-------------------------------------------------------------------- +%% Test Cases +%%-------------------------------------------------------------------- +t_log_and_pub(_) -> + %% Sub topic first + SubBase = "/test", + emqx:subscribe("$slow_topics"), + Clients = start_client(SubBase), + timer:sleep(1000), + Now = ?NOW, + %% publish + ?assert(ets:info(?LOG_TAB, size) =:= 0), + lists:foreach(fun(I) -> + Topic = list_to_binary(io_lib:format("~s~p", [SubBase, I])), + Msg = emqx_message:make(Topic, <<"Hello">>), + emqx:publish(Msg#message{timestamp = Now - 1000}) + end, + lists:seq(1, 10)), + + ?assert(ets:info(?LOG_TAB, size) =:= 5), + + timer:sleep(2400), + + ?assert(ets:info(?LOG_TAB, size) =:= 0), + ?assert(ets:info(?TOPK_TAB, size) =:= 3), + try_receive(3), + try_receive(2), + [Client ! stop || Client <- Clients], + ok. + +base_conf() -> + [{top_k_num, 3}, + {threshold_time, 10}, + {notice_qos, 0}, + {notice_batch_size, 3}, + {notice_topic,"$slow_topics"}, + {time_window, 2000}, + {max_log_num, 5}]. + +start_client(Base) -> + [spawn(fun() -> + Topic = list_to_binary(io_lib:format("~s~p", [Base, I])), + client(Topic) + end) + || I <- lists:seq(1, 10)]. + +client(Topic) -> + {ok, C} = emqtt:start_link([{host, "localhost"}, + {clientid, Topic}, + {username, <<"plain">>}, + {password, <<"plain">>}]), + {ok, _} = emqtt:connect(C), + {ok, _, _} = emqtt:subscribe(C, Topic), + receive + stop -> + ok + end. + +try_receive(L) -> + receive + {deliver, _, #message{payload = Payload}} -> + #{<<"logs">> := Logs} = emqx_json:decode(Payload, [return_maps]), + ?assertEqual(length(Logs), L) + after 500 -> + ?assert(false) + end. diff --git a/rebar.config.erl b/rebar.config.erl index 1000a2c92..38dcb681b 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -303,6 +303,7 @@ relx_plugin_apps(ReleaseType) -> , emqx_recon , emqx_rule_engine , emqx_sasl + , emqx_st_statistics ] ++ [emqx_telemetry || not is_enterprise()] ++ relx_plugin_apps_per_rel(ReleaseType) diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 596f7404a..7c2ea5eb2 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -5,38 +5,64 @@ latest_release=$(git describe --abbrev=0 --tags) bad_app_count=0 -while read -r app; do - if [ "$app" != "emqx" ]; then - app_path="$app" - else - app_path="." - fi - src_file="$app_path/src/$(basename "$app").app.src" - old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"')" - now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') - if [ "$old_app_version" = "$now_app_version" ]; then - changed="$(git diff --name-only "$latest_release"...HEAD \ - -- "$app_path/src" \ - -- "$app_path/priv" \ - -- "$app_path/c_src" | { grep -v -E 'appup\.src' || true; } | wc -l)" - if [ "$changed" -gt 0 ]; then - echo "$src_file needs a vsn bump" - bad_app_count=$(( bad_app_count + 1)) - elif [[ ${app_path} = *emqx_dashboard* ]]; then - ## emqx_dashboard is ensured to be upgraded after all other plugins - ## at the end of its appup instructions, there is the final instruction - ## {apply, {emqx_plugins, load, []} - ## since we don't know which plugins are stopped during the upgrade - ## for safty, we just force a dashboard version bump for each and every release - ## even if there is nothing changed in the app - echo "$src_file needs a vsn bump to ensure plugins loaded after upgrade" - bad_app_count=$(( bad_app_count + 1)) +get_vsn() { + commit="$1" + app_src_file="$2" + if [ "$commit" = 'HEAD' ]; then + if [ -f "$app_src_file" ]; then + grep vsn "$app_src_file" | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"' || true fi + else + git show "$commit":"$app_src_file" 2>/dev/null | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"' || true fi -done < <(./scripts/find-apps.sh) +} -if [ $bad_app_count -gt 0 ]; then - exit 1 -else - echo "apps version check successfully" -fi +check_apps() { + while read -r app_path; do + app=$(basename "$app_path") + src_file="$app_path/src/$app.app.src" + old_app_version="$(get_vsn "$latest_release" "$src_file")" + ## TODO: delete it after new version is released with emqx app in apps dir + if [ "$app" = 'emqx' ] && [ "$old_app_version" = '' ]; then + old_app_version="$(get_vsn "$latest_release" 'src/emqx.app.src')" + fi + now_app_version="$(get_vsn 'HEAD' "$src_file")" + ## TODO: delete it after new version is released with emqx app in apps dir + if [ "$app" = 'emqx' ] && [ "$now_app_version" = '' ]; then + now_app_version="$(get_vsn 'HEAD' 'src/emqx.app.src')" + fi + if [ -z "$now_app_version" ]; then + echo "failed_to_get_new_app_vsn for $app" + exit 1 + fi + if [ -z "${old_app_version:-}" ]; then + echo "skiped checking new app ${app}" + elif [ "$old_app_version" = "$now_app_version" ]; then + lines="$(git diff --name-only "$latest_release"...HEAD \ + -- "$app_path/src" \ + -- "$app_path/priv" \ + -- "$app_path/c_src")" + if [ "$lines" != '' ]; then + echo "$src_file needs a vsn bump (old=$old_app_version)" + echo "changed: $lines" + bad_app_count=$(( bad_app_count + 1)) + fi + fi + done < <(./scripts/find-apps.sh) + + if [ $bad_app_count -gt 0 ]; then + exit 1 + else + echo "apps version check successfully" + fi +} + +_main() { + if echo "${latest_release}" |grep -oE '[0-9]+.[0-9]+.[0-9]+' > /dev/null 2>&1; then + check_apps + else + echo "skiped unstable tag: ${latest_release}" + fi +} + +_main diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 9463345d4..db09bf8c9 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -319,6 +319,7 @@ is_awaiting_full(#session{awaiting_rel = AwaitingRel, puback(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> + emqx:run_hook('message.publish_done', [Msg]), Inflight1 = emqx_inflight:delete(PacketId, Inflight), return_with(Msg, dequeue(Session#session{inflight = Inflight1})); {value, {_Pubrel, _Ts}} -> @@ -343,6 +344,8 @@ return_with(Msg, {ok, Publishes, Session}) -> pubrec(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> + %% execute hook here, because message record will be replaced by pubrel + emqx:run_hook('message.publish_done', [Msg]), Inflight1 = emqx_inflight:update(PacketId, with_ts(pubrel), Inflight), {ok, Msg, Session#session{inflight = Inflight1}}; {value, {pubrel, _Ts}} -> @@ -439,11 +442,12 @@ deliver([Msg | More], Acc, Session) -> end. deliver_msg(Msg = #message{qos = ?QOS_0}, Session) -> + emqx:run_hook('message.publish_done', [Msg]), {ok, [{undefined, maybe_ack(Msg)}], Session}; deliver_msg(Msg = #message{qos = QoS}, Session = - #session{next_pkt_id = PacketId, inflight = Inflight}) - when QoS =:= ?QOS_1 orelse QoS =:= ?QOS_2 -> + #session{next_pkt_id = PacketId, inflight = Inflight}) + when QoS =:= ?QOS_1 orelse QoS =:= ?QOS_2 -> case emqx_inflight:is_full(Inflight) of true -> Session1 = case maybe_nack(Msg) of @@ -696,4 +700,3 @@ age(Now, Ts) -> Now - Ts. set_field(Name, Value, Session) -> Pos = emqx_misc:index_of(Name, record_info(fields, session)), setelement(Pos+1, Session, Value). - From b9270ad7197fdeb50de646452e8cee136bec57c0 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 3 Nov 2021 17:45:48 -0300 Subject: [PATCH 004/104] feat(stats): track live / connected channel count for monitoring In order to correctly display the number of connected clients in our monitor dashboard, we need to track those connections that are actually connected to clients, not considering connections from persistent sessions that are disconnected. Today, the `connections.count` that is displayed in the dashboards considers those disconnected persistent sessions as well. --- src/emqx_channel.erl | 5 +- src/emqx_cm.erl | 39 +++- src/emqx_connection.erl | 6 +- src/emqx_stats.erl | 13 +- test/emqx_broker_SUITE.erl | 291 +++++++++++++++++++++++++++++- test/emqx_channel_SUITE.erl | 3 +- test/emqx_connection_SUITE.erl | 18 +- test/emqx_mqueue_SUITE.erl | 2 +- test/emqx_ws_connection_SUITE.erl | 11 +- 9 files changed, 359 insertions(+), 29 deletions(-) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 7bfef472d..5f164d99e 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -1536,6 +1536,8 @@ ensure_connected(Channel = #channel{conninfo = ConnInfo, clientinfo = ClientInfo}) -> NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, ok = run_hooks('client.connected', [ClientInfo, NConnInfo]), + ChanPid = self(), + emqx_cm:mark_channel_connected(ChanPid), Channel#channel{conninfo = NConnInfo, conn_state = connected }. @@ -1624,6 +1626,8 @@ ensure_disconnected(Reason, Channel = #channel{conninfo = ConnInfo, clientinfo = ClientInfo}) -> NConnInfo = ConnInfo#{disconnected_at => erlang:system_time(millisecond)}, ok = run_hooks('client.disconnected', [ClientInfo, Reason, NConnInfo]), + ChanPid = self(), + emqx_cm:mark_channel_disconnected(ChanPid), Channel#channel{conninfo = NConnInfo, conn_state = disconnected}. %%-------------------------------------------------------------------- @@ -1725,4 +1729,3 @@ flag(false) -> 0. set_field(Name, Value, Channel) -> Pos = emqx_misc:index_of(Name, record_info(fields, channel)), setelement(Pos+1, Channel, Value). - diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 23f078568..c54089e41 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -22,6 +22,7 @@ -include("emqx.hrl"). -include("logger.hrl"). -include("types.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -logger_header("[CM]"). @@ -72,7 +73,12 @@ ]). %% Internal export --export([stats_fun/0, clean_down/1]). +-export([ stats_fun/0 + , clean_down/1 + , mark_channel_connected/1 + , mark_channel_disconnected/1 + , get_connected_client_count/0 + ]). -type(chan_pid() :: pid()). @@ -80,11 +86,13 @@ -define(CHAN_TAB, emqx_channel). -define(CHAN_CONN_TAB, emqx_channel_conn). -define(CHAN_INFO_TAB, emqx_channel_info). +-define(CHAN_LIVE_TAB, emqx_channel_live). -define(CHAN_STATS, [{?CHAN_TAB, 'channels.count', 'channels.max'}, {?CHAN_TAB, 'sessions.count', 'sessions.max'}, - {?CHAN_CONN_TAB, 'connections.count', 'connections.max'} + {?CHAN_CONN_TAB, 'connections.count', 'connections.max'}, + {?CHAN_LIVE_TAB, 'live_connections.count', 'live_connections.max'} ]). %% Batch drain @@ -437,8 +445,10 @@ init([]) -> ok = emqx_tables:new(?CHAN_TAB, [bag, {read_concurrency, true}|TabOpts]), ok = emqx_tables:new(?CHAN_CONN_TAB, [bag | TabOpts]), ok = emqx_tables:new(?CHAN_INFO_TAB, [set, compressed | TabOpts]), + ok = emqx_tables:new(?CHAN_LIVE_TAB, [set, {write_concurrency, true} | TabOpts]), ok = emqx_stats:update_interval(chan_stats, fun ?MODULE:stats_fun/0), - {ok, #{chan_pmon => emqx_pmon:new()}}. + State = #{chan_pmon => emqx_pmon:new()}, + {ok, State}. handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), @@ -447,17 +457,21 @@ handle_call(Req, _From, State) -> handle_cast({registered, {ClientId, ChanPid}}, State = #{chan_pmon := PMon}) -> PMon1 = emqx_pmon:monitor(ChanPid, ClientId, PMon), {noreply, State#{chan_pmon := PMon1}}; - handle_cast(Msg, State) -> ?LOG(error, "Unexpected cast: ~p", [Msg]), {noreply, State}. handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}) -> + ?tp(emqx_cm_process_down, #{pid => Pid, reason => _Reason}), ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), + lists:foreach( + fun({ChanPid, _ClientID}) -> + mark_channel_disconnected(ChanPid) + end, + Items), ok = emqx_pool:async_submit(fun lists:foreach/2, [fun ?MODULE:clean_down/1, Items]), {noreply, State#{chan_pmon := PMon1}}; - handle_info(Info, State) -> ?LOG(error, "Unexpected info: ~p", [Info]), {noreply, State}. @@ -493,3 +507,18 @@ get_chann_conn_mod(ClientId, ChanPid) when node(ChanPid) == node() -> get_chann_conn_mod(ClientId, ChanPid) -> rpc_call(node(ChanPid), get_chann_conn_mod, [ClientId, ChanPid], ?T_GET_INFO). +mark_channel_connected(ChanPid) -> + ?tp(emqx_cm_connected_client_count_inc, #{}), + ets:insert_new(?CHAN_LIVE_TAB, {ChanPid, true}), + ok. + +mark_channel_disconnected(ChanPid) -> + ?tp(emqx_cm_connected_client_count_dec, #{}), + ets:delete(?CHAN_LIVE_TAB, ChanPid), + ok. + +get_connected_client_count() -> + case ets:info(?CHAN_LIVE_TAB, size) of + undefined -> 0; + Size -> Size + end. diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index ab91c02b4..84086015d 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -518,7 +518,7 @@ terminate(Reason, State = #state{channel = Channel, transport = Transport, ?tp(warning, unclean_terminate, #{exception => E, context => C, stacktrace => S}) end, ?tp(info, terminate, #{reason => Reason}), - maybe_raise_excption(Reason). + maybe_raise_exception(Reason). %% close socket, discard new state, always return ok. close_socket_ok(State) -> @@ -526,12 +526,12 @@ close_socket_ok(State) -> ok. %% tell truth about the original exception -maybe_raise_excption(#{exception := Exception, +maybe_raise_exception(#{exception := Exception, context := Context, stacktrace := Stacktrace }) -> erlang:raise(Exception, Context, Stacktrace); -maybe_raise_excption(Reason) -> +maybe_raise_exception(Reason) -> exit(Reason). %%-------------------------------------------------------------------- diff --git a/src/emqx_stats.erl b/src/emqx_stats.erl index f53549e65..ba61143ac 100644 --- a/src/emqx_stats.erl +++ b/src/emqx_stats.erl @@ -21,6 +21,7 @@ -include("emqx.hrl"). -include("logger.hrl"). -include("types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -logger_header("[Stats]"). @@ -67,8 +68,10 @@ %% Connection stats -define(CONNECTION_STATS, - ['connections.count', %% Count of Concurrent Connections - 'connections.max' %% Maximum Number of Concurrent Connections + [ 'connections.count' %% Count of Concurrent Connections + , 'connections.max' %% Maximum Number of Concurrent Connections + , 'live_connections.count' %% Count of connected clients + , 'live_connections.max' %% Maximum number of connected clients ]). %% Channel stats @@ -216,6 +219,11 @@ handle_cast({setstat, Stat, MaxStat, Val}, State) -> ets:insert(?TAB, {MaxStat, Val}) end, safe_update_element(Stat, Val), + ?tp(emqx_stats_setstat, + #{ count_stat => Stat + , max_stat => MaxStat + , value => Val + }), {noreply, State}; handle_cast({update_interval, Update = #update{name = Name}}, @@ -274,4 +282,3 @@ safe_update_element(Key, Val) -> error:badarg -> ?LOG(warning, "Failed to update ~0p to ~0p", [Key, Val]) end. - diff --git a/test/emqx_broker_SUITE.erl b/test/emqx_broker_SUITE.erl index 4cef68660..57f0d6acf 100644 --- a/test/emqx_broker_SUITE.erl +++ b/test/emqx_broker_SUITE.erl @@ -23,20 +23,63 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> + [ {group, all_cases} + , {group, connected_client_count_group} + ]. -init_per_suite(Config) -> +groups() -> + TCs = emqx_ct:all(?MODULE), + ConnClientTCs = [ t_connected_client_count_persistent + , t_connected_client_count_anonymous + , t_connected_client_stats + ], + OtherTCs = TCs -- ConnClientTCs, + [ {all_cases, [], OtherTCs} + , {connected_client_count_group, [ {group, tcp} + , {group, ws} + ]} + , {tcp, [], ConnClientTCs} + , {ws, [], ConnClientTCs} + ]. + +init_per_group(connected_client_count_group, Config) -> + Config; +init_per_group(tcp, Config) -> + emqx_ct_helpers:boot_modules(all), + emqx_ct_helpers:start_apps([]), + [{conn_fun, connect} | Config]; +init_per_group(ws, Config) -> + emqx_ct_helpers:boot_modules(all), + emqx_ct_helpers:start_apps([]), + [ {ssl, false} + , {enable_websocket, true} + , {conn_fun, ws_connect} + , {port, 8083} + , {host, "localhost"} + | Config + ]; +init_per_group(_Group, Config) -> emqx_ct_helpers:boot_modules(all), emqx_ct_helpers:start_apps([]), Config. -end_per_suite(_Config) -> +end_per_group(connected_client_count_group, _Config) -> + ok; +end_per_group(_Group, _Config) -> emqx_ct_helpers:stop_apps([]). +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + init_per_testcase(Case, Config) -> ?MODULE:Case({init, Config}). @@ -277,6 +320,248 @@ t_stats_fun({'end', _Config}) -> ok = emqx_broker:unsubscribe(<<"topic">>), ok = emqx_broker:unsubscribe(<<"topic2">>). +%% persistent sessions, when gone, do not contribute to connected +%% client count +t_connected_client_count_persistent({init, Config}) -> + ok = snabbkaffe:start_trace(), + process_flag(trap_exit, true), + Config; +t_connected_client_count_persistent(Config) when is_list(Config) -> + ConnFun = ?config(conn_fun, Config), + ClientID = <<"clientid">>, + ?assertEqual(0, emqx_cm:get_connected_client_count()), + {ok, ConnPid0} = emqtt:start_link([ {clean_start, false} + , {clientid, ClientID} + | Config]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid0) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + {ok, {ok, [_]}} = wait_for_events( + fun() -> emqtt:disconnect(ConnPid0) end, + [emqx_cm_connected_client_count_dec] + ), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% reconnecting + {ok, ConnPid1} = emqtt:start_link([ {clean_start, false} + , {clientid, ClientID} + | Config + ]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid1) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% taking over + {ok, ConnPid2} = emqtt:start_link([ {clean_start, false} + , {clientid, ClientID} + | Config + ]), + {{ok, _}, {ok, [_, _]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid2) end, + [ emqx_cm_connected_client_count_inc + , emqx_cm_connected_client_count_dec + ], + 500 + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% abnormal exit of channel process + ChanPids = emqx_cm:all_channels(), + {ok, {ok, [_, _]}} = wait_for_events( + fun() -> + lists:foreach( + fun(ChanPid) -> exit(ChanPid, kill) end, + ChanPids) + end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + ok; +t_connected_client_count_persistent({'end', _Config}) -> + snabbkaffe:stop(), + ok. + +%% connections without client_id also contribute to connected client +%% count +t_connected_client_count_anonymous({init, Config}) -> + ok = snabbkaffe:start_trace(), + process_flag(trap_exit, true), + Config; +t_connected_client_count_anonymous(Config) when is_list(Config) -> + ConnFun = ?config(conn_fun, Config), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% first client + {ok, ConnPid0} = emqtt:start_link([ {clean_start, true} + | Config]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid0) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% second client + {ok, ConnPid1} = emqtt:start_link([ {clean_start, true} + | Config]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid1) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(2, emqx_cm:get_connected_client_count()), + %% when first client disconnects, shouldn't affect the second + {ok, {ok, [_, _]}} = wait_for_events( + fun() -> emqtt:disconnect(ConnPid0) end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% reconnecting + {ok, ConnPid2} = emqtt:start_link([ {clean_start, true} + | Config + ]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid2) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(2, emqx_cm:get_connected_client_count()), + {ok, {ok, [_, _]}} = wait_for_events( + fun() -> emqtt:disconnect(ConnPid1) end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% abnormal exit of channel process + Chans = emqx_cm:all_channels(), + {ok, {ok, [_, _]}} = wait_for_events( + fun() -> + lists:foreach( + fun(ChanPid) -> exit(ChanPid, kill) end, + Chans) + end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + ok; +t_connected_client_count_anonymous({'end', _Config}) -> + snabbkaffe:stop(), + ok. + +t_connected_client_stats({init, Config}) -> + ok = supervisor:terminate_child(emqx_kernel_sup, emqx_stats), + {ok, _} = supervisor:restart_child(emqx_kernel_sup, emqx_stats), + ok = snabbkaffe:start_trace(), + Config; +t_connected_client_stats(Config) when is_list(Config) -> + ConnFun = ?config(conn_fun, Config), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + ?assertEqual(0, emqx_stats:getstat('live_connections.count')), + ?assertEqual(0, emqx_stats:getstat('live_connections.max')), + {ok, ConnPid} = emqtt:start_link([ {clean_start, true} + , {clientid, <<"clientid">>} + | Config + ]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid) end, + [emqx_cm_connected_client_count_inc] + ), + %% ensure stats are synchronized + {_, {ok, [_]}} = wait_for_stats( + fun emqx_cm:stats_fun/0, + [#{count_stat => 'live_connections.count', + max_stat => 'live_connections.max'}] + ), + ?assertEqual(1, emqx_stats:getstat('live_connections.count')), + ?assertEqual(1, emqx_stats:getstat('live_connections.max')), + {ok, {ok, [_]}} = wait_for_events( + fun() -> emqtt:disconnect(ConnPid) end, + [emqx_cm_connected_client_count_dec] + ), + %% ensure stats are synchronized + {_, {ok, [_]}} = wait_for_stats( + fun emqx_cm:stats_fun/0, + [#{count_stat => 'live_connections.count', + max_stat => 'live_connections.max'}] + ), + ?assertEqual(0, emqx_stats:getstat('live_connections.count')), + ?assertEqual(1, emqx_stats:getstat('live_connections.max')), + ok; +t_connected_client_stats({'end', _Config}) -> + ok = snabbkaffe:stop(), + ok = supervisor:terminate_child(emqx_kernel_sup, emqx_stats), + {ok, _} = supervisor:restart_child(emqx_kernel_sup, emqx_stats), + ok. + +%% the count must be always non negative +t_connect_client_never_negative({init, Config}) -> + Config; +t_connect_client_never_negative(Config) when is_list(Config) -> + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% would go to -1 + ChanPid = list_to_pid("<0.0.1>"), + emqx_cm:mark_channel_disconnected(ChanPid), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% would be 0, if really went to -1 + emqx_cm:mark_channel_connected(ChanPid), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + ok; +t_connect_client_never_negative({'end', _Config}) -> + ok. + +wait_for_events(Action, Kinds) -> + wait_for_events(Action, Kinds, 500). + +wait_for_events(Action, Kinds, Timeout) -> + Predicate = fun(#{?snk_kind := K}) -> + lists:member(K, Kinds) + end, + N = length(Kinds), + {ok, Sub} = snabbkaffe_collector:subscribe(Predicate, N, Timeout, 0), + Res = Action(), + case snabbkaffe_collector:receive_events(Sub) of + {timeout, _} -> + {Res, timeout}; + {ok, Events} -> + {Res, {ok, Events}} + end. + +wait_for_stats(Action, Stats) -> + Predicate = fun(Event = #{?snk_kind := emqx_stats_setstat}) -> + Stat = maps:with( + [ count_stat + , max_stat + ], Event), + lists:member(Stat, Stats); + (_) -> + false + end, + N = length(Stats), + Timeout = 500, + {ok, Sub} = snabbkaffe_collector:subscribe(Predicate, N, Timeout, 0), + Res = Action(), + case snabbkaffe_collector:receive_events(Sub) of + {timeout, _} -> + {Res, timeout}; + {ok, Events} -> + {Res, {ok, Events}} + end. + +insert_fake_channels() -> + %% Insert copies to simulate missed counts + Tab = emqx_channel_info, + Key = ets:first(Tab), + [{_Chan, ChanInfo = #{conn_state := connected}, Stats}] = ets:lookup(Tab, Key), + ets:insert(Tab, [ {{"fake" ++ integer_to_list(N), undefined}, ChanInfo, Stats} + || N <- lists:seq(1, 9)]), + %% these should not be counted + ets:insert(Tab, [ { {"fake" ++ integer_to_list(N), undefined} + , ChanInfo#{conn_state := disconnected}, Stats} + || N <- lists:seq(10, 20)]). + recv_msgs(Count) -> recv_msgs(Count, []). diff --git a/test/emqx_channel_SUITE.erl b/test/emqx_channel_SUITE.erl index 9558dfd28..60e2541aa 100644 --- a/test/emqx_channel_SUITE.erl +++ b/test/emqx_channel_SUITE.erl @@ -33,6 +33,8 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> %% CM Meck ok = meck:new(emqx_cm, [passthrough, no_history, no_link]), + ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end), + ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end), %% Access Control Meck ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), ok = meck:expect(emqx_access_control, authenticate, @@ -835,4 +837,3 @@ session(InitFields) when is_map(InitFields) -> quota() -> emqx_limiter:init(zone, [{conn_messages_routing, {5, 1}}, {overall_messages_routing, {10, 1}}]). - diff --git a/test/emqx_connection_SUITE.erl b/test/emqx_connection_SUITE.erl index a6b2b614a..3dcc280d4 100644 --- a/test/emqx_connection_SUITE.erl +++ b/test/emqx_connection_SUITE.erl @@ -36,6 +36,8 @@ init_per_suite(Config) -> ok = meck:new(emqx_channel, [passthrough, no_history, no_link]), %% Meck Cm ok = meck:new(emqx_cm, [passthrough, no_history, no_link]), + ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end), + ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end), %% Meck Limiter ok = meck:new(emqx_limiter, [passthrough, no_history, no_link]), %% Meck Pd @@ -112,7 +114,7 @@ t_ws_pingreq_before_connected(_) -> t_info(_) -> CPid = spawn(fun() -> - receive + receive {'$gen_call', From, info} -> gen_server:reply(From, emqx_connection:info(st())) after @@ -132,7 +134,7 @@ t_info_limiter(_) -> t_stats(_) -> CPid = spawn(fun() -> - receive + receive {'$gen_call', From, stats} -> gen_server:reply(From, emqx_connection:stats(st())) after @@ -147,10 +149,10 @@ t_stats(_) -> {send_pend,0}| _] , Stats). t_process_msg(_) -> - with_conn(fun(CPid) -> - ok = meck:expect(emqx_channel, handle_in, - fun(_Packet, Channel) -> - {ok, Channel} + with_conn(fun(CPid) -> + ok = meck:expect(emqx_channel, handle_in, + fun(_Packet, Channel) -> + {ok, Channel} end), CPid ! {incoming, ?PACKET(?PINGREQ)}, CPid ! {incoming, undefined}, @@ -318,7 +320,7 @@ t_with_channel(_) -> t_handle_outgoing(_) -> ?assertEqual(ok, emqx_connection:handle_outgoing(?PACKET(?PINGRESP), st())), ?assertEqual(ok, emqx_connection:handle_outgoing([?PACKET(?PINGRESP)], st())). - + t_handle_info(_) -> ?assertMatch({ok, {event,running}, _NState}, emqx_connection:handle_info(activate_socket, st())), @@ -345,7 +347,7 @@ t_activate_socket(_) -> State = st(), {ok, NStats} = emqx_connection:activate_socket(State), ?assertEqual(running, emqx_connection:info(sockstate, NStats)), - + State1 = st(#{sockstate => blocked}), ?assertEqual({ok, State1}, emqx_connection:activate_socket(State1)), diff --git a/test/emqx_mqueue_SUITE.erl b/test/emqx_mqueue_SUITE.erl index 599e68da1..52b93dceb 100644 --- a/test/emqx_mqueue_SUITE.erl +++ b/test/emqx_mqueue_SUITE.erl @@ -121,7 +121,7 @@ t_priority_mqueue(_) -> ?assertEqual(5, ?Q:len(Q5)), {_, Q6} = ?Q:in(#message{qos = 1, topic = <<"t2">>}, Q5), ?assertEqual(5, ?Q:len(Q6)), - {{value, Msg}, Q7} = ?Q:out(Q6), + {{value, _Msg}, Q7} = ?Q:out(Q6), ?assertEqual(4, ?Q:len(Q7)). t_priority_mqueue_conservation(_) -> diff --git a/test/emqx_ws_connection_SUITE.erl b/test/emqx_ws_connection_SUITE.erl index 56d038c23..3c1aad3de 100644 --- a/test/emqx_ws_connection_SUITE.erl +++ b/test/emqx_ws_connection_SUITE.erl @@ -48,6 +48,10 @@ init_per_testcase(TestCase, Config) when TestCase =/= t_ws_pingreq_before_connected, TestCase =/= t_ws_non_check_origin -> + %% Meck Cm + ok = meck:new(emqx_cm, [passthrough, no_history, no_link]), + ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end), + ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end), %% Mock cowboy_req ok = meck:new(cowboy_req, [passthrough, no_history, no_link]), ok = meck:expect(cowboy_req, header, fun(_, _, _) -> <<>> end), @@ -95,7 +99,8 @@ end_per_testcase(TestCase, _Config) when TestCase =/= t_ws_pingreq_before_connected -> lists:foreach(fun meck:unload/1, - [cowboy_req, + [emqx_cm, + cowboy_req, emqx_zone, emqx_access_control, emqx_broker, @@ -389,14 +394,12 @@ t_handle_info_close(_) -> {[{close, _}], _St} = ?ws_conn:handle_info({close, protocol_error}, st()). t_handle_info_event(_) -> - ok = meck:new(emqx_cm, [passthrough, no_history]), ok = meck:expect(emqx_cm, register_channel, fun(_,_,_) -> ok end), ok = meck:expect(emqx_cm, insert_channel_info, fun(_,_,_) -> ok end), ok = meck:expect(emqx_cm, connection_closed, fun(_) -> true end), {ok, _} = ?ws_conn:handle_info({event, connected}, st()), {ok, _} = ?ws_conn:handle_info({event, disconnected}, st()), - {ok, _} = ?ws_conn:handle_info({event, updated}, st()), - ok = meck:unload(emqx_cm). + {ok, _} = ?ws_conn:handle_info({event, updated}, st()). t_handle_timeout_idle_timeout(_) -> TRef = make_ref(), From fec83590a6aa5d984bdc50a6e5dc2d862e96213e Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 8 Nov 2021 21:38:18 +0100 Subject: [PATCH 005/104] chore: bump release version to 4.4.0 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index c89dde010..24f6feb57 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.9"}). +-define(EMQX_RELEASE, {opensource, "4.4.0"}). -else. From 9ea5c5e58df3bcd8376c81cbbaf89adad7927439 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 8 Nov 2021 23:11:32 +0100 Subject: [PATCH 006/104] fix(emqx_mgmt): support v4.4 data export --- apps/emqx_management/include/emqx_mgmt.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl index 6d510ed0c..26c566f50 100644 --- a/apps/emqx_management/include/emqx_mgmt.hrl +++ b/apps/emqx_management/include/emqx_mgmt.hrl @@ -32,4 +32,4 @@ -define(ERROR14, 114). %% OldPassword error -define(ERROR15, 115). %% bad topic --define(VERSIONS, ["4.0", "4.1", "4.2", "4.3"]). \ No newline at end of file +-define(VERSIONS, ["4.0", "4.1", "4.2", "4.3", "4.4"]). From 00ba4d32f3351235c2f1a6a933da2fd22dc18246 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 3 Nov 2021 11:19:37 +0800 Subject: [PATCH 007/104] fix(syntax): allow single quotes in the FROM clause --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 16f7193bb..a38d74e88 100644 --- a/rebar.config +++ b/rebar.config @@ -51,7 +51,7 @@ , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3.1"}}} - , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.2"}}} + , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} From 2242bb93763bcd9c32b98db9e99b22529f946b89 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 9 Nov 2021 08:30:57 +0800 Subject: [PATCH 008/104] fix(rule): force max speed to 2 decimal digits of precision --- apps/emqx_rule_engine/src/emqx_rule_metrics.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl index bc2b04c07..c88533f6f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl @@ -410,7 +410,7 @@ calculate_speed(CurrVal, #rule_speed{max = MaxSpeed0, last_v = LastVal, last5m_smpl = Last5MinSamples, tick = Tick + 1}. format_rule_speed(#rule_speed{max = Max, current = Current, last5m = Last5Min}) -> - #{max => Max, current => precision(Current, 2), last5m => precision(Last5Min, 2)}. + #{max => precision(Max, 2), current => precision(Current, 2), last5m => precision(Last5Min, 2)}. precision(Float, N) -> Base = math:pow(10, N), From 77c82cf1898ff790d359e34775ae6d91bcafd28b Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 9 Nov 2021 08:39:36 +0800 Subject: [PATCH 009/104] fix(code_style): some elvis complaints --- apps/emqx_rule_engine/src/emqx_rule_metrics.erl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl index c88533f6f..57ba6e3da 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl @@ -73,6 +73,8 @@ , terminate/2 ]). +-elvis([{elvis_style, god_modules, disable}]). + -ifndef(TEST). -define(SECS_5M, 300). -define(SAMPLING, 10). @@ -235,7 +237,7 @@ start_link() -> init([]) -> erlang:process_flag(trap_exit, true), %% the overall counters - [ok = emqx_metrics:ensure(Metric)|| Metric <- overall_metrics()], + [ok = emqx_metrics:ensure(Metric) || Metric <- overall_metrics()], %% the speed metrics erlang:send_after(timer:seconds(?SAMPLING), self(), ticking), {ok, #state{overall_rule_speed = #rule_speed{}}}. @@ -388,17 +390,19 @@ calculate_speed(CurrVal, #rule_speed{max = MaxSpeed0, last_v = LastVal, %% calculate the max speed since the emqx startup MaxSpeed = - if MaxSpeed0 >= CurrSpeed -> MaxSpeed0; - true -> CurrSpeed + case MaxSpeed0 >= CurrSpeed of + true -> MaxSpeed0; + false -> CurrSpeed end, %% calculate the average speed in last 5 mins {Last5MinSamples, Acc5Min, Last5Min} = - if Tick =< ?SAMPCOUNT_5M -> + case Tick =< ?SAMPCOUNT_5M of + true -> Acc = AccSpeed5Min0 + CurrSpeed, {lists:reverse([CurrSpeed | lists:reverse(Last5MinSamples0)]), Acc, Acc / Tick}; - true -> + false -> [FirstSpeed | Speeds] = Last5MinSamples0, Acc = AccSpeed5Min0 + CurrSpeed - FirstSpeed, {lists:reverse([CurrSpeed | lists:reverse(Speeds)]), From 354b0bc08e24edf99b7867eadf07eea29f55f00a Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 5 Nov 2021 16:11:48 +0800 Subject: [PATCH 010/104] refactor(emqx_st_statistics): optimize the implementation of topk --- .../src/emqx_st_statistics.appup.src | 9 - .../src/emqx_st_statistics.erl | 199 ++++++++++-------- .../test/emqx_st_statistics_SUITE.erl | 9 +- 3 files changed, 118 insertions(+), 99 deletions(-) delete mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics.appup.src diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.appup.src b/apps/emqx_st_statistics/src/emqx_st_statistics.appup.src deleted file mode 100644 index dcf0d8cdd..000000000 --- a/apps/emqx_st_statistics/src/emqx_st_statistics.appup.src +++ /dev/null @@ -1,9 +0,0 @@ -%% -*-: erlang -*- -{VSN, - [ - {<<".*">>, []} - ], - [ - {<<".*">>, []} - ] -}. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.erl b/apps/emqx_st_statistics/src/emqx_st_statistics.erl index 8a6c1535f..801a21c96 100644 --- a/apps/emqx_st_statistics/src/emqx_st_statistics.erl +++ b/apps/emqx_st_statistics/src/emqx_st_statistics.erl @@ -40,28 +40,30 @@ -compile(nowarn_unused_type). -type state() :: #{ config := proplist:proplist() - , index := index_map() - , begin_time := pos_integer() + , period := pos_integer() + , last_tick_at := pos_integer() , counter := counters:counter_ref() , enable := boolean() }. -type log() :: #{ topic := emqx_types:topic() - , times := pos_integer() + , count := pos_integer() , average := float() }. --type window_log() :: #{ begin_time := pos_integer() +-type window_log() :: #{ last_tick_at := pos_integer() , logs := [log()] }. -record(slow_log, { topic :: emqx_types:topic() - , times :: non_neg_integer() - , elapsed :: non_neg_integer() + , count :: pos_integer() + , elapsed :: pos_integer() }). --record(top_k, { key :: any() - , average :: float()}). +-record(top_k, { rank :: pos_integer() + , topic :: emqx_types:topic() + , average_count :: number() + , average_elapsed :: number()}). -type message() :: #message{}. @@ -70,11 +72,11 @@ -define(LOG_TAB, emqx_st_statistics_log). -define(TOPK_TAB, emqx_st_statistics_topk). -define(NOW, erlang:system_time(millisecond)). --define(TOP_KEY(Times, Topic), {Times, Topic}). -define(QUOTA_IDX, 1). --type top_key() :: ?TOP_KEY(pos_integer(), emqx_types:topic()). --type index_map() :: #{emqx_types:topic() => pos_integer()}. +-type slow_log() :: #slow_log{}. +-type top_k() :: #top_k{}. +-type top_k_map() :: #{emqx_types:topic() => top_k()}. %% erlang term order %% number < atom < reference < fun < port < pid < tuple < list < bit string @@ -124,8 +126,8 @@ init([Env]) -> Threshold = get_value(threshold_time, Env), load(Threshold, Counter), {ok, #{config => Env, - index => #{}, - begin_time => ?NOW, + period => 1, + last_tick_at => ?NOW, counter => Counter, enable => true}}. @@ -152,11 +154,11 @@ handle_cast(Msg, State) -> ?LOG(error, "Unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info(notification_tick, #{config := Cfg} = State) -> +handle_info(notification_tick, #{config := Cfg, period := Period} = State) -> notification_tick(Cfg), - Index2 = do_notification(State), - {noreply, State#{index := Index2, - begin_time := ?NOW}}; + do_notification(State), + {noreply, State#{last_tick_at := ?NOW, + period := Period + 1}}; handle_info(Info, State) -> ?LOG(error, "Unexpected info: ~p", [Info]), @@ -183,9 +185,9 @@ init_log_tab(_) -> ]). init_topk_tab(_) -> - ?TOPK_TAB = ets:new(?TOPK_TAB, [ ordered_set, protected, named_table - , {keypos, #top_k.key}, {write_concurrency, true} - , {read_concurrency, false} + ?TOPK_TAB = ets:new(?TOPK_TAB, [ set, protected, named_table + , {keypos, #top_k.rank}, {write_concurrency, false} + , {read_concurrency, true} ]). -spec get_log_quota(counter:counter_ref()) -> boolean(). @@ -203,107 +205,104 @@ set_log_quota(Cfg, Counter) -> MaxLogNum = get_value(max_log_num, Cfg), counters:put(Counter, ?QUOTA_IDX, MaxLogNum). --spec update_log(message(), non_neg_integer()) -> ok. +-spec update_log(message(), pos_integer()) -> ok. update_log(#message{topic = Topic}, Elapsed) -> _ = ets:update_counter(?LOG_TAB, Topic, - [{#slow_log.times, 1}, {#slow_log.elapsed, Elapsed}], + [{#slow_log.count, 1}, {#slow_log.elapsed, Elapsed}], #slow_log{topic = Topic, - times = 1, - elapsed = Elapsed}), + count = 1, + elapsed = Elapsed}), ok. --spec do_notification(state()) -> index_map(). -do_notification(#{begin_time := BeginTime, +-spec do_notification(state()) -> true. +do_notification(#{last_tick_at := TickTime, config := Cfg, - index := IndexMap, + period := Period, counter := Counter}) -> Logs = ets:tab2list(?LOG_TAB), ets:delete_all_objects(?LOG_TAB), - start_publish(Logs, BeginTime, Cfg), + start_publish(Logs, TickTime, Cfg), set_log_quota(Cfg, Counter), MaxRecord = get_value(top_k_num, Cfg), - Size = ets:info(?TOPK_TAB, size), - update_top_k(Logs, erlang:max(0, MaxRecord - Size), IndexMap). + update_topk(Logs, MaxRecord, Period). --spec update_top_k(list(#slow_log{}), non_neg_integer(), index_map()) -> index_map(). -update_top_k([#slow_log{topic = Topic, - times = NewTimes, - elapsed = Elapsed} = Log | T], - Left, - IndexMap) -> - case maps:get(Topic, IndexMap, 0) of - 0 -> - try_insert_new(Log, Left, T, IndexMap); - Times -> - [#top_k{key = Key, average = Average}] = ets:lookup(?TOPK_TAB, ?TOP_KEY(Times, Topic)), - Times2 = Times + NewTimes, - Total = Times * Average + Elapsed, - Average2 = Total / Times2, - ets:delete(?TOPK_TAB, Key), - ets:insert(?TOPK_TAB, #top_k{key = ?TOP_KEY(Times2, Topic), average = Average2}), - update_top_k(T, Left, IndexMap#{Topic := Times2}) +-spec update_topk(list(slow_log()), pos_integer(), pos_integer()) -> true. +update_topk(Logs, MaxRecord, Period) -> + TopkMap = get_topk_map(Period), + TopkMap2 = update_topk_map(Logs, Period, TopkMap), + SortFun = fun(A, B) -> + A#top_k.average_count > B#top_k.average_count + end, + TopkL = lists:sort(SortFun, maps:values(TopkMap2)), + TopkL2 = lists:sublist(TopkL, 1, MaxRecord), + update_topk_tab(TopkL2). + +-spec update_topk_map(list(slow_log()), pos_integer(), top_k_map()) -> top_k_map(). +update_topk_map([#slow_log{topic = Topic, + count = LogTimes, + elapsed = LogElapsed} | T], Period, TopkMap) -> + case maps:get(Topic, TopkMap, undefined) of + undefined -> + Record = #top_k{rank = 1, + topic = Topic, + average_count = LogTimes, + average_elapsed = LogElapsed}, + TopkMap2 = TopkMap#{Topic => Record}, + update_topk_map(T, Period, TopkMap2); + #top_k{average_count = AvgCount, + average_elapsed = AvgElapsed} = Record -> + NewPeriod = Period + 1, + %% (a + b) / c = a / c + b / c + %% average_count(elapsed) dived NewPeriod in function get_topk_maps + AvgCount2 = AvgCount + LogTimes / NewPeriod, + AvgElapsed2 = AvgElapsed + LogElapsed / NewPeriod, + Record2 = Record#top_k{average_count = AvgCount2, + average_elapsed = AvgElapsed2}, + update_topk_map(T, Period, TopkMap#{Topic := Record2}) end; -update_top_k([], _, IndexMap) -> - IndexMap. +update_topk_map([], _, TopkMap) -> + TopkMap. --spec try_insert_new(#slow_log{}, - non_neg_integer(), list(#slow_log{}), index_map()) -> index_map(). -try_insert_new(#slow_log{topic = Topic, - times = Times, - elapsed = Elapsed}, Left, Logs, IndexMap) when Left > 0 -> - Average = Elapsed / Times, - ets:insert_new(?TOPK_TAB, #top_k{key = ?TOP_KEY(Times, Topic), average = Average}), - update_top_k(Logs, Left - 1, IndexMap#{Topic => Times}); +-spec update_topk_tab(list(top_k())) -> true. +update_topk_tab(Records) -> + Zip = fun(Rank, Item) -> Item#top_k{rank = Rank} end, + Len = erlang:length(Records), + RankedTopics = lists:zipwith(Zip, lists:seq(1, Len), Records), + ets:insert(?TOPK_TAB, RankedTopics). -try_insert_new(#slow_log{topic = Topic, - times = Times, - elapsed = Elapsed}, Left, Logs, IndexMap) -> - ?TOP_KEY(MinTimes, MinTopic) = MinKey = ets:first(?TOPK_TAB), - case MinTimes > Times of - true -> - update_top_k(Logs, Left, IndexMap); - _ -> - Average = Elapsed / Times, - ets:delete(?TOPK_TAB, MinKey), - ets:insert_new(?TOPK_TAB, #top_k{key = ?TOP_KEY(Times, Topic), average = Average}), - update_top_k(Logs, - Left - 1, - maps:put(Topic, Times, maps:remove(MinTopic, IndexMap))) - end. - -start_publish(Logs, BeginTime, Cfg) -> - emqx_pool:async_submit({fun do_publish/3, [Logs, BeginTime, Cfg]}). +start_publish(Logs, TickTime, Cfg) -> + emqx_pool:async_submit({fun do_publish/3, [Logs, TickTime, Cfg]}). do_publish([], _, _) -> ok; -do_publish(Logs, BeginTime, Cfg) -> +do_publish(Logs, TickTime, Cfg) -> BatchSize = get_value(notice_batch_size, Cfg), - do_publish(Logs, BatchSize, BeginTime, Cfg, []). + do_publish(Logs, BatchSize, TickTime, Cfg, []). -do_publish([Log | T], Size, BeginTime, Cfg, Cache) when Size > 0 -> +do_publish([Log | T], Size, TickTime, Cfg, Cache) when Size > 0 -> Cache2 = [convert_to_notice(Log) | Cache], - do_publish(T, Size - 1, BeginTime, Cfg, Cache2); + do_publish(T, Size - 1, TickTime, Cfg, Cache2); -do_publish(Logs, Size, BeginTime, Cfg, Cache) when Size =:= 0 -> - publish(BeginTime, Cfg, Cache), - do_publish(Logs, BeginTime, Cfg); +do_publish(Logs, Size, TickTime, Cfg, Cache) when Size =:= 0 -> + publish(TickTime, Cfg, Cache), + do_publish(Logs, TickTime, Cfg); -do_publish([], _, BeginTime, Cfg, Cache) -> - publish(BeginTime, Cfg, Cache), +do_publish([], _, TickTime, Cfg, Cache) -> + publish(TickTime, Cfg, Cache), ok. convert_to_notice(#slow_log{topic = Topic, - times = Times, + count = Count, elapsed = Elapsed}) -> #{topic => Topic, - times => Times, - average => Elapsed / Times}. + count => Count, + average => Elapsed / Count}. -publish(BeginTime, Cfg, Notices) -> - WindowLog = #{begin_time => BeginTime, +publish(TickTime, Cfg, Notices) -> + WindowLog = #{last_tick_at => TickTime, logs => Notices}, Payload = emqx_json:encode(WindowLog), _ = emqx:publish(#message{ id = emqx_guid:gen() @@ -322,6 +321,7 @@ load(Threshold, Counter) -> unload() -> emqx:unhook('message.publish_done', fun ?MODULE:on_publish_done/3). +-spec get_topic(proplists:proplist()) -> binary(). get_topic(Cfg) -> case get_value(notice_topic, Cfg) of Topic when is_binary(Topic) -> @@ -329,3 +329,24 @@ get_topic(Cfg) -> Topic -> erlang:list_to_binary(Topic) end. + +-spec get_topk_map(pos_integer()) -> top_k_map(). +get_topk_map(Period) -> + Size = ets:info(?TOPK_TAB, size), + get_topk_map(1, Size, Period, #{}). + +-spec get_topk_map(pos_integer(), + non_neg_integer(), pos_integer(), top_k_map()) -> top_k_map(). +get_topk_map(Index, Size, _, TopkMap) when Index > Size -> + TopkMap; +get_topk_map(Index, Size, Period, TopkMap) -> + [#top_k{topic = Topic, + average_count = AvgCount, + average_elapsed = AvgElapsed} = R] = ets:lookup(?TOPK_TAB, Index), + NewPeriod = Period + 1, + TotalTimes = AvgCount * Period, + AvgCount2 = TotalTimes / NewPeriod, + AvgElapsed2 = TotalTimes * AvgElapsed / NewPeriod, + TopkMap2 = TopkMap#{Topic => R#top_k{average_count = AvgCount2, + average_elapsed = AvgElapsed2}}, + get_topk_map(Index + 1, Size, Period, TopkMap2). diff --git a/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl b/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl index c08c8d986..e8bdd129f 100644 --- a/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl +++ b/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl @@ -61,7 +61,14 @@ t_log_and_pub(_) -> end, lists:seq(1, 10)), - ?assert(ets:info(?LOG_TAB, size) =:= 5), + timer:sleep(100), + + case ets:info(?LOG_TAB, size) of + 5 -> + ok; + _ -> + ?assert(ets:info(?TOPK_TAB, size) =/= 0) + end, timer:sleep(2400), From 388c29344a6ad7133f0a55480de9d6618a054962 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 10 Nov 2021 09:51:49 +0800 Subject: [PATCH 011/104] fix(relup): configs for plugins are missing after relup --- bin/install_upgrade.escript | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index 0658bdef2..97548cba8 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -248,15 +248,20 @@ parse_version(V) when is_list(V) -> hd(string:tokens(V,"/")). check_and_install(TargetNode, Vsn) -> - {ok, [[CurrAppConf]]} = rpc:call(TargetNode, init, get_argument, [config], ?TIMEOUT), + %% Backup the vm.args. VM args should be unchanged during hot upgrade + %% but we still backup it here {ok, [[CurrVmArgs]]} = rpc:call(TargetNode, init, get_argument, [vm_args], ?TIMEOUT), - case filename:extension(CurrAppConf) of - ".config" -> - {ok, _} = file:copy(CurrAppConf, filename:join(["releases", Vsn, "sys.config"])); - _ -> - {ok, _} = file:copy(CurrAppConf++".config", filename:join(["releases", Vsn, "sys.config"])) - end, {ok, _} = file:copy(CurrVmArgs, filename:join(["releases", Vsn, "vm.args"])), + %% Backup the sys.config, this will be used when we check and install release + %% NOTE: We cannot backup the old sys.config directly, because the + %% configs for plugins are only in app-envs, not in the old sys.config + Configs0 = + [{AppName, rpc:call(TargetNode, application, get_all_env, [AppName], ?TIMEOUT)} + || {AppName, _, _} <- rpc:call(TargetNode, application, which_applications, [], ?TIMEOUT)], + Configs1 = [{AppName, Conf} || {AppName, Conf} <- Configs0, Conf =/= []], + ok = file:write_file(filename:join(["releases", Vsn, "sys.config"]), io_lib:format("~p.", [Configs1])), + + %% check and install release case rpc:call(TargetNode, release_handler, check_install_release, [Vsn], ?TIMEOUT) of {ok, _OtherVsn, _Desc} -> From fae815b35c283434121cbb3eba3c5be83f0e587c Mon Sep 17 00:00:00 2001 From: lafirest Date: Wed, 10 Nov 2021 11:08:06 +0800 Subject: [PATCH 012/104] Feat/slow topic api (#6101) * feat(emqx_st_statistics): add api --- .../include/emqx_st_statistics.hrl | 25 +++ .../src/emqx_st_statistics.erl | 17 +- .../src/emqx_st_statistics_api.erl | 75 +++++++++ .../src/emqx_st_statistics_app.erl | 8 +- .../test/emqx_st_statistics_api_SUITE.erl | 152 ++++++++++++++++++ .../src/emqx_mod_st_statistics.erl | 47 ++++++ lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- .../emqx_modules/src/emqx_modules.appup.src | 6 + priv/emqx.schema | 1 + 9 files changed, 328 insertions(+), 5 deletions(-) create mode 100644 apps/emqx_st_statistics/include/emqx_st_statistics.hrl create mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics_api.erl create mode 100644 apps/emqx_st_statistics/test/emqx_st_statistics_api_SUITE.erl create mode 100644 lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl diff --git a/apps/emqx_st_statistics/include/emqx_st_statistics.hrl b/apps/emqx_st_statistics/include/emqx_st_statistics.hrl new file mode 100644 index 000000000..9184c3194 --- /dev/null +++ b/apps/emqx_st_statistics/include/emqx_st_statistics.hrl @@ -0,0 +1,25 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-define(LOG_TAB, emqx_st_statistics_log). +-define(TOPK_TAB, emqx_st_statistics_topk). + +-record(top_k, { rank :: pos_integer() + , topic :: emqx_types:topic() + , average_count :: number() + , average_elapsed :: number()}). + +-type top_k() :: #top_k{}. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.erl b/apps/emqx_st_statistics/src/emqx_st_statistics.erl index 801a21c96..2aa5c9fb4 100644 --- a/apps/emqx_st_statistics/src/emqx_st_statistics.erl +++ b/apps/emqx_st_statistics/src/emqx_st_statistics.erl @@ -25,7 +25,7 @@ -logger_header("[SLOW TOPICS]"). -export([ start_link/1, on_publish_done/3, enable/0 - , disable/0 + , disable/0, clear_history/0 ]). %% gen_server callbacks @@ -78,6 +78,12 @@ -type top_k() :: #top_k{}. -type top_k_map() :: #{emqx_types:topic() => top_k()}. +-ifdef(TEST). +-define(TOPK_ACCESS, public). +-else. +-define(TOPK_ACCESS, protected). +-endif. + %% erlang term order %% number < atom < reference < fun < port < pid < tuple < list < bit string @@ -106,6 +112,9 @@ on_publish_done(#message{timestamp = Timestamp} = Msg, Threshold, Counter) -> ok end. +clear_history() -> + gen_server:call(?MODULE, ?FUNCTION_NAME). + enable() -> gen_server:call(?MODULE, {enable, true}). @@ -146,6 +155,10 @@ handle_call({enable, Enable}, _From, end, {reply, ok, State2}; +handle_call(clear_history, _, State) -> + ets:delete_all_objects(?TOPK_TAB), + {reply, ok, State}; + handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), {reply, ignored, State}. @@ -185,7 +198,7 @@ init_log_tab(_) -> ]). init_topk_tab(_) -> - ?TOPK_TAB = ets:new(?TOPK_TAB, [ set, protected, named_table + ?TOPK_TAB = ets:new(?TOPK_TAB, [ set, ?TOPK_ACCESS, named_table , {keypos, #top_k.rank}, {write_concurrency, false} , {read_concurrency, true} ]). diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics_api.erl b/apps/emqx_st_statistics/src/emqx_st_statistics_api.erl new file mode 100644 index 000000000..3980caccd --- /dev/null +++ b/apps/emqx_st_statistics/src/emqx_st_statistics_api.erl @@ -0,0 +1,75 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_st_statistics_api). + +-rest_api(#{name => clear_history, + method => 'DELETE', + path => "/slow_topic", + func => clear_history, + descr => "Clear current data and re count slow topic"}). + +-rest_api(#{name => get_history, + method => 'GET', + path => "/slow_topic", + func => get_history, + descr => "Get slow topics statistics record data"}). + +-export([ clear_history/2 + , get_history/2 + ]). + +-include_lib("emqx_st_statistics/include/emqx_st_statistics.hrl"). + +-import(minirest, [return/1]). + +%%-------------------------------------------------------------------- +%% HTTP API +%%-------------------------------------------------------------------- + +clear_history(_Bindings, _Params) -> + ok = emqx_st_statistics:clear_history(), + return(ok). + +get_history(_Bindings, Params) -> + PageT = proplists:get_value(<<"_page">>, Params), + LimitT = proplists:get_value(<<"_limit">>, Params), + Page = erlang:binary_to_integer(PageT), + Limit = erlang:binary_to_integer(LimitT), + Start = (Page - 1) * Limit + 1, + Size = ets:info(?TOPK_TAB, size), + EndT = Start + Limit - 1, + End = erlang:min(EndT, Size), + Infos = lists:foldl(fun(Rank, Acc) -> + [#top_k{topic = Topic + , average_count = Count + , average_elapsed = Elapsed}] = ets:lookup(?TOPK_TAB, Rank), + + Info =[ {rank, Rank} + , {topic, Topic} + , {count, Count} + , {elapsed, Elapsed}], + + [Info | Acc] + end, + [], + lists:seq(Start, End)), + + return({ok, #{meta => #{page => Page, + limit => Limit, + hasnext => End < Size, + count => End - Start + 1}, + data => Infos}}). diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl b/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl index e5b62a00e..b8e5987db 100644 --- a/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl +++ b/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl @@ -22,11 +22,15 @@ -export([ start/2 , stop/1 + , start/0 ]). start(_Type, _Args) -> - Env = application:get_all_env(emqx_st_statistics), - {ok, Sup} = emqx_st_statistics_sup:start_link(Env), + start(). + +start() -> + Conf = application:get_all_env(emqx_st_statistics), + {ok, Sup} = emqx_st_statistics_sup:start_link(Conf), {ok, Sup}. stop(_State) -> diff --git a/apps/emqx_st_statistics/test/emqx_st_statistics_api_SUITE.erl b/apps/emqx_st_statistics/test/emqx_st_statistics_api_SUITE.erl new file mode 100644 index 000000000..e6b99e70a --- /dev/null +++ b/apps/emqx_st_statistics/test/emqx_st_statistics_api_SUITE.erl @@ -0,0 +1,152 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_st_statistics_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx_management/include/emqx_mgmt.hrl"). +-include_lib("emqx_st_statistics/include/emqx_st_statistics.hrl"). + +-define(CONTENT_TYPE, "application/x-www-form-urlencoded"). + +-define(HOST, "http://127.0.0.1:8081/"). + +-define(API_VERSION, "v4"). + +-define(BASE_PATH, "api"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx_modules), + emqx_ct_helpers:start_apps([emqx_st_statistics, emqx_management]), + Config. + +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx_st_statistics, emqx_management]), + Config. + +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(_, Config) -> + Config. + +get(Key, ResponseBody) -> + maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])). + +lookup_alarm(Name, [#{<<"name">> := Name} | _More]) -> + true; +lookup_alarm(Name, [_Alarm | More]) -> + lookup_alarm(Name, More); +lookup_alarm(_Name, []) -> + false. + +is_existing(Name, [#{name := Name} | _More]) -> + true; +is_existing(Name, [_Alarm | More]) -> + is_existing(Name, More); +is_existing(_Name, []) -> + false. + +t_get_history(_) -> + ets:insert(?TOPK_TAB, #top_k{rank = 1, + topic = <<"test">>, + average_count = 12, + average_elapsed = 1500}), + + {ok, Data} = request_api(get, api_path(["slow_topic"]), "_page=1&_limit=10", + auth_header_()), + + Return = #{meta => #{page => 1, + limit => 10, + hasnext => false, + count => 1}, + data => [#{topic => <<"test">>, + rank => 1, + elapsed => 1500, + count => 12}], + code => 0}, + + ShouldBe = emqx_json:encode(Return), + + ?assertEqual(ShouldBe, erlang:list_to_binary(Data)). + +t_clear(_) -> + ets:insert(?TOPK_TAB, #top_k{rank = 1, + topic = <<"test">>, + average_count = 12, + average_elapsed = 1500}), + + {ok, _} = request_api(delete, api_path(["slow_topic"]), [], + auth_header_()), + + ?assertEqual(0, ets:info(?TOPK_TAB, size)). + +request_api(Method, Url, Auth) -> + request_api(Method, Url, [], Auth, []). + +request_api(Method, Url, QueryParams, Auth) -> + request_api(Method, Url, QueryParams, Auth, []). + +request_api(Method, Url, QueryParams, Auth, []) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth]}); +request_api(Method, Url, QueryParams, Auth, Body) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + +do_request_api(Method, Request)-> + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], []) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{"HTTP/1.1", Code, _}, _, Return} } + when Code =:= 200 orelse Code =:= 201 -> + {ok, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +auth_header_() -> + AppId = <<"admin">>, + AppSecret = <<"public">>, + auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)). + +auth_header_(User, Pass) -> + Encoded = base64:encode_to_string(lists:append([User,":",Pass])), + {"Authorization","Basic " ++ Encoded}. + +api_path(Parts)-> + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts). + +filter(List, Key, Value) -> + lists:filter(fun(Item) -> + maps:get(Key, Item) == Value + end, List). diff --git a/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl b/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl new file mode 100644 index 000000000..01a1444bd --- /dev/null +++ b/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mod_st_statistics). + +-behaviour(emqx_gen_mod). + +-include_lib("emqx/include/logger.hrl"). + +-logger_header("[st_statistics]"). + +%% emqx_gen_mod callbacks +-export([ load/1 + , unload/1 + , description/0 + ]). + +%%-------------------------------------------------------------------- +%% Load/Unload +%%-------------------------------------------------------------------- + +-spec(load(list()) -> ok). +load(_Env) -> + _ = emqx_st_statistics_app:start(), + ok. + +-spec(unload(list()) -> ok). +unload(_Env) -> + emqx_st_statistics_app:stop(undefined). + +description() -> + "EMQ X Slow Topic Statistics Module". + +%%-------------------------------------------------------------------- diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index bcb05fe31..47a3d8888 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.3.3"}, + {vsn, "4.3.4"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index f52ba1a61..aed228213 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,6 +1,9 @@ %% -*-: erlang -*- {VSN, [ + {"4.3.3", [ + {load_module, emqx_mod_st_statistics, brutal_purge, soft_purge, []} + ]}, {"4.3.2", [ {load_module, emqx_mod_presence, brutal_purge, soft_purge, []} ]}, @@ -16,6 +19,9 @@ {<<".*">>, []} ], [ + {"4.3.3", [ + {load_module, emqx_mod_st_statistics, brutal_purge, soft_purge, []} + ]}, {"4.3.2", [ {load_module, emqx_mod_presence, brutal_purge, soft_purge, []} ]}, diff --git a/priv/emqx.schema b/priv/emqx.schema index 4b33cf65c..6418501e3 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -2217,6 +2217,7 @@ end}. [{emqx_mod_rewrite, Rewrites()}], [{emqx_mod_topic_metrics, []}], [{emqx_mod_delayed, []}], + [{emqx_mod_st_statistics, []}], [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] ]) end}. From 06a1b37992494fa0b6c22ac82ab7479993186c8b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 10 Nov 2021 15:41:31 +0800 Subject: [PATCH 013/104] fix(test): flaky mqtt expiry test case. (#6111) --- test/emqx_mqtt_SUITE.erl | 112 +++++++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 39 deletions(-) diff --git a/test/emqx_mqtt_SUITE.erl b/test/emqx_mqtt_SUITE.erl index ae7e7382c..3a77134b8 100644 --- a/test/emqx_mqtt_SUITE.erl +++ b/test/emqx_mqtt_SUITE.erl @@ -62,79 +62,104 @@ t_conn_stats(_) -> t_tcp_sock_passive(_) -> with_client(fun(CPid) -> CPid ! {tcp_passive, sock} end, []). -t_message_expiry_interval_1(_) -> - ClientA = message_expiry_interval_init(), - [message_expiry_interval_exipred(ClientA, QoS) || QoS <- [0,1,2]], - emqtt:stop(ClientA). +t_message_expiry_interval(_) -> + {CPublish, CControl} = message_expiry_interval_init(), + [message_expiry_interval_exipred(CPublish, CControl, QoS) || QoS <- [0,1,2]], + emqtt:stop(CPublish), + emqtt:stop(CControl). -t_message_expiry_interval_2(_) -> - ClientA = message_expiry_interval_init(), - [message_expiry_interval_not_exipred(ClientA, QoS) || QoS <- [0,1,2]], - emqtt:stop(ClientA). +t_message_not_expiry_interval(_) -> + {CPublish, CControl} = message_expiry_interval_init(), + [message_expiry_interval_not_exipred(CPublish, CControl, QoS) || QoS <- [0,1,2]], + emqtt:stop(CPublish), + emqtt:stop(CControl). message_expiry_interval_init() -> - {ok, ClientA} = emqtt:start_link([{proto_ver,v5}, - {clientid, <<"client-a">>}, + {ok, CPublish} = emqtt:start_link([{proto_ver,v5}, + {clientid, <<"Client-Publish">>}, {clean_start, false}, {properties, #{'Session-Expiry-Interval' => 360}}]), - {ok, ClientB} = emqtt:start_link([{proto_ver,v5}, - {clientid, <<"client-b">>}, + {ok, CVerify} = emqtt:start_link([{proto_ver,v5}, + {clientid, <<"Client-Verify">>}, {clean_start, false}, {properties, #{'Session-Expiry-Interval' => 360}}]), - {ok, _} = emqtt:connect(ClientA), - {ok, _} = emqtt:connect(ClientB), - %% subscribe and disconnect client-b - emqtt:subscribe(ClientB, <<"t/a">>, 1), - emqtt:stop(ClientB), - ClientA. + {ok, CControl} = emqtt:start_link([{proto_ver,v5}, + {clientid, <<"Client-Control">>}, + {clean_start, false}, + {properties, #{'Session-Expiry-Interval' => 360}}]), + {ok, _} = emqtt:connect(CPublish), + {ok, _} = emqtt:connect(CVerify), + {ok, _} = emqtt:connect(CControl), + %% subscribe and disconnect Client-verify + emqtt:subscribe(CControl, <<"t/a">>, 1), + emqtt:subscribe(CVerify, <<"t/a">>, 1), + emqtt:stop(CVerify), + {CPublish, CControl}. -message_expiry_interval_exipred(ClientA, QoS) -> +message_expiry_interval_exipred(CPublish, CControl, QoS) -> ct:pal("~p ~p", [?FUNCTION_NAME, QoS]), %% publish to t/a and waiting for the message expired - emqtt:publish(ClientA, <<"t/a">>, #{'Message-Expiry-Interval' => 1}, <<"this will be purged in 1s">>, [{qos, QoS}]), - ct:sleep(2000), + emqtt:publish(CPublish, <<"t/a">>, #{'Message-Expiry-Interval' => 1}, + <<"this will be purged in 1s">>, [{qos, QoS}]), + %% CControl make sure publish already store in broker. + receive + {publish,#{client_pid := CControl, topic := <<"t/a">>}} -> + ok + after 1000 -> + ct:fail(should_receive_publish) + end, + ct:sleep(1100), - %% resume the session for client-b - {ok, ClientB1} = emqtt:start_link([{proto_ver,v5}, - {clientid, <<"client-b">>}, + %% resume the session for Client-Verify + {ok, CVerify} = emqtt:start_link([{proto_ver,v5}, + {clientid, <<"Client-Verify">>}, {clean_start, false}, {properties, #{'Session-Expiry-Interval' => 360}}]), - {ok, _} = emqtt:connect(ClientB1), + {ok, _} = emqtt:connect(CVerify), - %% verify client-b could not receive the publish message + %% verify Client-Verify could not receive the publish message receive - {publish,#{client_pid := ClientB1, topic := <<"t/a">>}} -> + {publish,#{client_pid := CVerify, topic := <<"t/a">>}} -> ct:fail(should_have_expired) after 300 -> ok end, - emqtt:stop(ClientB1). + emqtt:stop(CVerify). -message_expiry_interval_not_exipred(ClientA, QoS) -> +message_expiry_interval_not_exipred(CPublish, CControl, QoS) -> ct:pal("~p ~p", [?FUNCTION_NAME, QoS]), %% publish to t/a - emqtt:publish(ClientA, <<"t/a">>, #{'Message-Expiry-Interval' => 20}, <<"this will be purged in 1s">>, [{qos, QoS}]), + emqtt:publish(CPublish, <<"t/a">>, #{'Message-Expiry-Interval' => 20}, + <<"this will be purged in 20s">>, [{qos, QoS}]), - %% wait for 1s and then resume the session for client-b, the message should not expires + %% CControl make sure publish already store in broker. + receive + {publish,#{client_pid := CControl, topic := <<"t/a">>}} -> + ok + after 1000 -> + ct:fail(should_receive_publish) + end, + + %% wait for 1.2s and then resume the session for Client-Verify, the message should not expires %% as Message-Expiry-Interval = 20s - ct:sleep(1000), - {ok, ClientB1} = emqtt:start_link([{proto_ver,v5}, - {clientid, <<"client-b">>}, + ct:sleep(1200), + {ok, CVerify} = emqtt:start_link([{proto_ver,v5}, + {clientid, <<"Client-Verify">>}, {clean_start, false}, {properties, #{'Session-Expiry-Interval' => 360}}]), - {ok, _} = emqtt:connect(ClientB1), + {ok, _} = emqtt:connect(CVerify), - %% verify client-b could receive the publish message and the Message-Expiry-Interval is set + %% verify Client-Verify could receive the publish message and the Message-Expiry-Interval is set receive - {publish,#{client_pid := ClientB1, topic := <<"t/a">>, + {publish,#{client_pid := CVerify, topic := <<"t/a">>, properties := #{'Message-Expiry-Interval' := MsgExpItvl}}} - when MsgExpItvl < 20 -> ok; + when MsgExpItvl =< 20 -> ok; {publish, _} = Msg -> ct:fail({incorrect_publish, Msg}) after 300 -> ct:fail(no_publish_received) end, - emqtt:stop(ClientB1). + emqtt:stop(CVerify). with_client(TestFun, _Options) -> ClientId = <<"t_conn">>, @@ -156,6 +181,15 @@ t_async_set_keepalive('end', _Config) -> ok. t_async_set_keepalive(_) -> + case os:type() of + {unix, darwin} -> + %% Mac OSX don't support the feature + ok; + _ -> + do_async_set_keepalive() + end. + +do_async_set_keepalive() -> ClientID = <<"client-tcp-keepalive">>, {ok, Client} = emqtt:start_link([{host, "localhost"}, {proto_ver,v5}, From 1dd18aa07a2168ed3fd706c70aa14a5b07290833 Mon Sep 17 00:00:00 2001 From: lafirest Date: Thu, 11 Nov 2021 10:16:06 +0800 Subject: [PATCH 014/104] =?UTF-8?q?fix(emqx=5Fst=5Fstatistics):=20change?= =?UTF-8?q?=20emqx=5Fst=5Fstatistics=20implementation=20fro=E2=80=A6=20(#6?= =?UTF-8?q?115)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(emqx_st_statistics): change emqx_st_statistics implementation from plugin to module --- .../etc/emqx_st_statistics.conf | 38 ------------ .../priv/emqx_st_statistics.schema | 58 ------------------- apps/emqx_st_statistics/rebar.config | 23 -------- .../src/emqx_st_statistics.app.src | 12 ---- .../src/emqx_st_statistics_app.erl | 37 ------------ .../src/emqx_st_statistics_sup.erl | 35 ----------- etc/emqx.conf | 38 ++++++++++++ .../include/emqx_st_statistics.hrl | 0 .../src/emqx_mod_st_statistics.erl | 47 --------------- lib-ce/emqx_modules/src/emqx_mod_sup.erl | 21 ++++--- .../emqx_mod_st_statistics.erl | 40 +++++++++---- .../emqx_st_statistics_api.erl | 4 +- .../test/emqx_st_statistics_SUITE.erl | 16 +++-- .../test/emqx_st_statistics_api_SUITE.erl | 8 ++- priv/emqx.schema | 44 +++++++++++++- rebar.config.erl | 1 - 16 files changed, 140 insertions(+), 282 deletions(-) delete mode 100644 apps/emqx_st_statistics/etc/emqx_st_statistics.conf delete mode 100644 apps/emqx_st_statistics/priv/emqx_st_statistics.schema delete mode 100644 apps/emqx_st_statistics/rebar.config delete mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics.app.src delete mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics_app.erl delete mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl rename {apps/emqx_st_statistics => lib-ce/emqx_modules}/include/emqx_st_statistics.hrl (100%) delete mode 100644 lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl rename apps/emqx_st_statistics/src/emqx_st_statistics.erl => lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl (94%) rename {apps/emqx_st_statistics/src => lib-ce/emqx_modules/src/emqx_st_statistics}/emqx_st_statistics_api.erl (96%) rename {apps/emqx_st_statistics => lib-ce/emqx_modules}/test/emqx_st_statistics_SUITE.erl (93%) rename {apps/emqx_st_statistics => lib-ce/emqx_modules}/test/emqx_st_statistics_api_SUITE.erl (94%) diff --git a/apps/emqx_st_statistics/etc/emqx_st_statistics.conf b/apps/emqx_st_statistics/etc/emqx_st_statistics.conf deleted file mode 100644 index 88ca0c37f..000000000 --- a/apps/emqx_st_statistics/etc/emqx_st_statistics.conf +++ /dev/null @@ -1,38 +0,0 @@ -##-------------------------------------------------------------------- -## EMQ X Slow Topics Statistics -##-------------------------------------------------------------------- - -## Threshold time of slow topics statistics -## -## Default: 10 seconds -st_statistics.threshold_time = 10S - -## Time window of slow topics statistics -## -## Value: 5 minutes -st_statistics.time_window = 5M - -## Maximum of slow topics log, log will clear when enter new time window -## -## Value: 500 -st_statistics.max_log_num = 500 - -## Top-K record for slow topics, update from logs -## -## Value: 500 -st_statistics.top_k_num = 500 - -## Topic of notification -## -## Defaut: $slow_topics -st_statistics.notice_topic = $slow_topics - -## QoS of notification message in notice topic -## -## Defaut: 0 -st_statistics.notice_qos = 0 - -## Maximum information number in one notification -## -## Default: 500 -st_statistics.notice_batch_size = 500 diff --git a/apps/emqx_st_statistics/priv/emqx_st_statistics.schema b/apps/emqx_st_statistics/priv/emqx_st_statistics.schema deleted file mode 100644 index da9a2f810..000000000 --- a/apps/emqx_st_statistics/priv/emqx_st_statistics.schema +++ /dev/null @@ -1,58 +0,0 @@ -%%-*- mode: erlang -*- -%% st_statistics config mapping - -%% Threshold time of slow topics statistics -%% {$configurable} -{mapping, "st_statistics.threshold_time", "emqx_st_statistics.threshold_time", - [ - {default, "10S"}, - {datatype, [integer, {duration, ms}]} - ]}. - -%% Time window of slow topics statistics -%% {$configurable} -{mapping, "st_statistics.time_window", "emqx_st_statistics.time_window", - [ - {default, "5M"}, - {datatype, [integer, {duration, ms}]} - ]}. - -%% Maximum of slow topics log -%% {$configurable} -{mapping, "st_statistics.max_log_num", "emqx_st_statistics.max_log_num", - [ - {default, 500}, - {datatype, integer} - ]}. - -%% Top-K record for slow topics, update from logs -%% {$configurable} -{mapping, "st_statistics.top_k_num", "emqx_st_statistics.top_k_num", - [ - {default, 500}, - {datatype, integer} - ]}. - -%% Topic of notification -%% {$configurable} -{mapping, "st_statistics.notice_topic", "emqx_st_statistics.notice_topic", - [ - {default, <<"slow_topics">>}, - {datatype, string} - ]}. - -%% QoS of notification message in notice topic -%% {$configurable} -{mapping, "st_statistics.notice_qos", "emqx_st_statistics.notice_qos", - [ - {default, 0}, - {datatype, integer} - ]}. - -%% Maximum entities per notification message -%% {$configurable} -{mapping, "st_statistics.notice_batch_size", "emqx_st_statistics.notice_batch_size", - [ - {default, 500}, - {datatype, integer} - ]}. diff --git a/apps/emqx_st_statistics/rebar.config b/apps/emqx_st_statistics/rebar.config deleted file mode 100644 index 6433d92d6..000000000 --- a/apps/emqx_st_statistics/rebar.config +++ /dev/null @@ -1,23 +0,0 @@ -{deps, []}. - -{edoc_opts, [{preprocess, true}]}. -{erl_opts, [warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard - ]}. - -{xref_checks, [undefined_function_calls, undefined_functions, - locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. -{cover_enabled, true}. -{cover_opts, [verbose]}. -{cover_export_enabled, true}. - -{profiles, - [{test, - [{deps, - [{emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}, - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3"}}}]} - ]} - ]}. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.app.src b/apps/emqx_st_statistics/src/emqx_st_statistics.app.src deleted file mode 100644 index b1eb1612a..000000000 --- a/apps/emqx_st_statistics/src/emqx_st_statistics.app.src +++ /dev/null @@ -1,12 +0,0 @@ -{application, emqx_st_statistics, - [{description, "EMQ X Slow Topics Statistics"}, - {vsn, "1.0.0"}, % strict semver, bump manually! - {modules, []}, - {registered, [emqx_st_statistics_sup]}, - {applications, [kernel,stdlib]}, - {mod, {emqx_st_statistics_app,[]}}, - {env, []}, - {licenses, ["Apache-2.0"]}, - {maintainers, ["EMQ X Team "]}, - {links, []} - ]}. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl b/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl deleted file mode 100644 index b8e5987db..000000000 --- a/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl +++ /dev/null @@ -1,37 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_st_statistics_app). - --behaviour(application). - --emqx_plugin(?MODULE). - --export([ start/2 - , stop/1 - , start/0 - ]). - -start(_Type, _Args) -> - start(). - -start() -> - Conf = application:get_all_env(emqx_st_statistics), - {ok, Sup} = emqx_st_statistics_sup:start_link(Conf), - {ok, Sup}. - -stop(_State) -> - ok. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl b/apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl deleted file mode 100644 index cbff854ef..000000000 --- a/apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl +++ /dev/null @@ -1,35 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_st_statistics_sup). - --behaviour(supervisor). - --export([start_link/1]). - --export([init/1]). - -start_link(Env) -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]). - -init([Env]) -> - {ok, {{one_for_one, 10, 3600}, - [#{id => st_statistics, - start => {emqx_st_statistics, start_link, [Env]}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_st_statistics]}]}}. diff --git a/etc/emqx.conf b/etc/emqx.conf index cf1279de1..a94afd961 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -2211,6 +2211,44 @@ module.presence.qos = 1 ## module.rewrite.pub.rule.1 = x/# ^x/y/(.+)$ z/y/$1 ## module.rewrite.sub.rule.1 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2 +##-------------------------------------------------------------------- +## Slow Topic Statistics Module + +## Threshold time of slow topics statistics +## +## Default: 10 seconds +#module.st_statistics.threshold_time = 10S + +## Time window of slow topics statistics +## +## Value: 5 minutes +#module.st_statistics.time_window = 5M + +## Maximum of slow topics log, log will clear when enter new time window +## +## Value: 500 +#module.st_statistics.max_log_num = 500 + +## Top-K record for slow topics, update from logs +## +## Value: 500 +#module.st_statistics.top_k_num = 500 + +## Topic of notification +## +## Defaut: $slow_topics +#module.st_statistics.notice_topic = $slow_topics + +## QoS of notification message in notice topic +## +## Defaut: 0 +#module.st_statistics.notice_qos = 0 + +## Maximum information number in one notification +## +## Default: 500 +#module.st_statistics.notice_batch_size = 500 + ## CONFIG_SECTION_END=modules ================================================== ##------------------------------------------------------------------- diff --git a/apps/emqx_st_statistics/include/emqx_st_statistics.hrl b/lib-ce/emqx_modules/include/emqx_st_statistics.hrl similarity index 100% rename from apps/emqx_st_statistics/include/emqx_st_statistics.hrl rename to lib-ce/emqx_modules/include/emqx_st_statistics.hrl diff --git a/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl b/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl deleted file mode 100644 index 01a1444bd..000000000 --- a/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl +++ /dev/null @@ -1,47 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mod_st_statistics). - --behaviour(emqx_gen_mod). - --include_lib("emqx/include/logger.hrl"). - --logger_header("[st_statistics]"). - -%% emqx_gen_mod callbacks --export([ load/1 - , unload/1 - , description/0 - ]). - -%%-------------------------------------------------------------------- -%% Load/Unload -%%-------------------------------------------------------------------- - --spec(load(list()) -> ok). -load(_Env) -> - _ = emqx_st_statistics_app:start(), - ok. - --spec(unload(list()) -> ok). -unload(_Env) -> - emqx_st_statistics_app:stop(undefined). - -description() -> - "EMQ X Slow Topic Statistics Module". - -%%-------------------------------------------------------------------- diff --git a/lib-ce/emqx_modules/src/emqx_mod_sup.erl b/lib-ce/emqx_modules/src/emqx_mod_sup.erl index 755e52a60..60907f580 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_sup.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_sup.erl @@ -23,18 +23,22 @@ -export([ start_link/0 , start_child/1 , start_child/2 + , start_child/3 , stop_child/1 ]). -export([init/1]). %% Helper macro for declaring children of supervisor --define(CHILD(Mod, Type), #{id => Mod, - start => {Mod, start_link, []}, - restart => permanent, - shutdown => 5000, - type => Type, - modules => [Mod]}). +-define(CHILD(Mod, Type, Args), + #{id => Mod, + start => {Mod, start_link, Args}, + restart => permanent, + shutdown => 5000, + type => Type, + modules => [Mod]}). + +-define(CHILD(MOD, Type), ?CHILD(MOD, Type, [])). -spec(start_link() -> startlink_ret()). start_link() -> @@ -48,6 +52,10 @@ start_child(ChildSpec) when is_map(ChildSpec) -> start_child(Mod, Type) when is_atom(Mod) andalso is_atom(Type) -> assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Type))). +-spec start_child(atom(), atom(), list(any())) -> ok. +start_child(Mod, Type, Args) when is_atom(Mod) andalso is_atom(Type) -> + assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Type, Args))). + -spec(stop_child(any()) -> ok | {error, term()}). stop_child(ChildId) -> case supervisor:terminate_child(?MODULE, ChildId) of @@ -71,4 +79,3 @@ assert_started({ok, _Pid}) -> ok; assert_started({ok, _Pid, _Info}) -> ok; assert_started({error, {already_tarted, _Pid}}) -> ok; assert_started({error, Reason}) -> erlang:error(Reason). - diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.erl b/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl similarity index 94% rename from apps/emqx_st_statistics/src/emqx_st_statistics.erl rename to lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl index 2aa5c9fb4..a66975767 100644 --- a/apps/emqx_st_statistics/src/emqx_st_statistics.erl +++ b/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -14,13 +14,14 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_st_statistics). +-module(emqx_mod_st_statistics). +-behaviour(emqx_gen_mod). -behaviour(gen_server). -include_lib("include/emqx.hrl"). -include_lib("include/logger.hrl"). --include_lib("stdlib/include/ms_transform.hrl"). +-include("include/emqx_st_statistics.hrl"). -logger_header("[SLOW TOPICS]"). @@ -28,6 +29,12 @@ , disable/0, clear_history/0 ]). +%% emqx_gen_mod callbacks +-export([ load/1 + , unload/1 + , description/0 + ]). + %% gen_server callbacks -export([ init/1 , handle_call/3 @@ -60,22 +67,14 @@ , elapsed :: pos_integer() }). --record(top_k, { rank :: pos_integer() - , topic :: emqx_types:topic() - , average_count :: number() - , average_elapsed :: number()}). - -type message() :: #message{}. -import(proplists, [get_value/2]). --define(LOG_TAB, emqx_st_statistics_log). --define(TOPK_TAB, emqx_st_statistics_topk). -define(NOW, erlang:system_time(millisecond)). -define(QUOTA_IDX, 1). -type slow_log() :: #slow_log{}. --type top_k() :: #top_k{}. -type top_k_map() :: #{emqx_types:topic() => top_k()}. -ifdef(TEST). @@ -90,9 +89,26 @@ %% ets ordered_set is ascending by term order %%-------------------------------------------------------------------- -%% APIs +%% Load/Unload %%-------------------------------------------------------------------- +-spec(load(list()) -> ok). +load(Env) -> + emqx_mod_sup:start_child(?MODULE, worker, [Env]), + ok. + +-spec(unload(list()) -> ok). +unload(_Env) -> + _ = emqx_mod_sup:stop_child(?MODULE), + ok. + +description() -> + "EMQ X Slow Topic Statistics Module". + +%%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- %% @doc Start the st_statistics -spec(start_link(Env :: list()) -> emqx_types:startlink_ret()). start_link(Env) -> diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics_api.erl b/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_st_statistics_api.erl similarity index 96% rename from apps/emqx_st_statistics/src/emqx_st_statistics_api.erl rename to lib-ce/emqx_modules/src/emqx_st_statistics/emqx_st_statistics_api.erl index 3980caccd..328fddf28 100644 --- a/apps/emqx_st_statistics/src/emqx_st_statistics_api.erl +++ b/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_st_statistics_api.erl @@ -32,7 +32,7 @@ , get_history/2 ]). --include_lib("emqx_st_statistics/include/emqx_st_statistics.hrl"). +-include("include/emqx_st_statistics.hrl"). -import(minirest, [return/1]). @@ -41,7 +41,7 @@ %%-------------------------------------------------------------------- clear_history(_Bindings, _Params) -> - ok = emqx_st_statistics:clear_history(), + ok = emqx_mod_st_statistics:clear_history(), return(ok). get_history(_Bindings, Params) -> diff --git a/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl b/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl similarity index 93% rename from apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl rename to lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl index e8bdd129f..36da813f2 100644 --- a/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl @@ -31,17 +31,21 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_st_statistics], fun set_special_cfg/1), + emqx_ct_helpers:start_apps([emqx]), Config. -set_special_cfg(_) -> - application:set_env([{emqx_st_statistics, base_conf()}]), - ok. - end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_st_statistics]), + emqx_ct_helpers:stop_apps([emqx]), Config. +init_per_testcase(_, Config) -> + emqx_mod_st_statistics:load(base_conf()), + Config. + +end_per_testcase(_, _) -> + emqx_mod_st_statistics:unload(undefined), + ok. + %%-------------------------------------------------------------------- %% Test Cases %%-------------------------------------------------------------------- diff --git a/apps/emqx_st_statistics/test/emqx_st_statistics_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl similarity index 94% rename from apps/emqx_st_statistics/test/emqx_st_statistics_api_SUITE.erl rename to lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl index e6b99e70a..8255cdd5f 100644 --- a/apps/emqx_st_statistics/test/emqx_st_statistics_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl @@ -24,7 +24,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx_management/include/emqx_mgmt.hrl"). --include_lib("emqx_st_statistics/include/emqx_st_statistics.hrl"). +-include_lib("emqx_modules/include/emqx_st_statistics.hrl"). -define(CONTENT_TYPE, "application/x-www-form-urlencoded"). @@ -39,17 +39,19 @@ all() -> init_per_suite(Config) -> application:load(emqx_modules), - emqx_ct_helpers:start_apps([emqx_st_statistics, emqx_management]), + emqx_ct_helpers:start_apps([emqx_management]), Config. end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_st_statistics, emqx_management]), + emqx_ct_helpers:stop_apps([emqx_management]), Config. init_per_testcase(_, Config) -> + emqx_mod_st_statistics:load(emqx_st_statistics_SUITE:base_conf()), Config. end_per_testcase(_, Config) -> + emqx_mod_st_statistics:unload(undefined), Config. get(Key, ResponseBody) -> diff --git a/priv/emqx.schema b/priv/emqx.schema index 6418501e3..febd4cb9e 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -2188,6 +2188,42 @@ end}. {datatype, string} ]}. +{mapping, "module.st_statistics.threshold_time", "emqx.modules", [ + {default, "10s"}, + {datatype, {duration, ms}} +]}. + +{mapping, "module.st_statistics.time_window", "emqx.modules", [ + {default, "5M"}, + {datatype, {duration, ms}} +]}. + +{mapping, "module.st_statistics.max_log_num", "emqx.modules", [ + {default, 500}, + {datatype, integer} +]}. + +{mapping, "module.st_statistics.top_k_num", "emqx.modules", [ + {default, 500}, + {datatype, integer} +]}. + +{mapping, "module.st_statistics.notice_topic", "emqx.modules", [ + {default, "$slow_topics"}, + {datatype, string} +]}. + +{mapping, "module.st_statistics.notice_qos", "emqx.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-1"]} +]}. + +{mapping, "module.st_statistics.notice_batch_size", "emqx.modules", [ + {default, 500}, + {datatype, integer} +]}. + {translation, "emqx.modules", fun(Conf, _, Conf1) -> Subscriptions = fun() -> List = cuttlefish_variable:filter_by_prefix("module.subscription", Conf), @@ -2211,13 +2247,19 @@ end}. {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)} end, TotalRules) end, + + SlowTopic = fun() -> + List = cuttlefish_variable:filter_by_prefix("module.st_statistics", Conf), + [{erlang:list_to_atom(Key), Value} || {[_, _, Key], Value} <- List] + end, + lists:append([ [{emqx_mod_presence, [{qos, cuttlefish:conf_get("module.presence.qos", Conf, 1)}]}], [{emqx_mod_subscription, Subscriptions()}], [{emqx_mod_rewrite, Rewrites()}], [{emqx_mod_topic_metrics, []}], [{emqx_mod_delayed, []}], - [{emqx_mod_st_statistics, []}], + [{emqx_mod_st_statistics, SlowTopic()}], [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] ]) end}. diff --git a/rebar.config.erl b/rebar.config.erl index 38dcb681b..1000a2c92 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -303,7 +303,6 @@ relx_plugin_apps(ReleaseType) -> , emqx_recon , emqx_rule_engine , emqx_sasl - , emqx_st_statistics ] ++ [emqx_telemetry || not is_enterprise()] ++ relx_plugin_apps_per_rel(ReleaseType) From 835ad52498efc79a62d795c183fce12585f47ddb Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 11 Nov 2021 14:54:40 +0800 Subject: [PATCH 015/104] feat(trace): http api for trace (#6052) * feat(trace): add http trace API * feat: sub/unsub trace topic * chore(trace): stream log use _page and _limit * fix: elvis warning * fix: mod_trace test failed * fix: http api meta rename _page/_limit to _page/_limit * fix: clientid string not working * fix: add emqx_mod_trace to module * fix(trace): fixed by review suggestions --- .../src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_mgmt_cli.erl | 122 +++-- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 26 +- deploy/charts/emqx/values.yaml | 2 + include/emqx_mqtt.hrl | 18 + lib-ce/emqx_modules/src/emqx_mod_sup.erl | 2 +- lib-ce/emqx_modules/src/emqx_mod_trace.erl | 508 ++++++++++++++++++ .../emqx_modules/src/emqx_mod_trace_api.erl | 191 +++++++ .../test/emqx_mod_rewrite_SUITE.erl | 2 +- .../emqx_modules/test/emqx_mod_sup_SUITE.erl | 3 +- .../test/emqx_mod_trace_SUITE.erl | 478 ++++++++++++++++ priv/emqx.schema | 1 + src/emqx_broker.erl | 50 +- src/emqx_guid.erl | 4 +- src/emqx_misc.erl | 38 +- src/emqx_packet.erl | 59 +- src/emqx_tracer.erl | 175 ++++-- test/emqx_tracer_SUITE.erl | 119 ++-- 18 files changed, 1580 insertions(+), 220 deletions(-) create mode 100644 lib-ce/emqx_modules/src/emqx_mod_trace.erl create mode 100644 lib-ce/emqx_modules/src/emqx_mod_trace_api.erl create mode 100644 lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 405b4c244..fe68fef44 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.8"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 95f5121cd..97cb182e6 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -21,7 +21,9 @@ -include("emqx_mgmt.hrl"). --define(PRINT_CMD(Cmd, Descr), io:format("~-48s# ~s~n", [Cmd, Descr])). +-elvis([{elvis_style, invalid_dynamic_call, #{ ignore => [emqx_mgmt_cli]}}]). + +-define(PRINT_CMD(Cmd, Desc), io:format("~-48s# ~s~n", [Cmd, Desc])). -export([load/0]). @@ -74,11 +76,8 @@ mgmt(["insert", AppId, Name]) -> mgmt(["lookup", AppId]) -> case emqx_mgmt_auth:lookup_app(list_to_binary(AppId)) of - {AppId1, AppSecret, Name, Desc, Status, Expired} -> - emqx_ctl:print("app_id: ~s~nsecret: ~s~nname: ~s~ndesc: ~s~nstatus: ~s~nexpired: ~p~n", - [AppId1, AppSecret, Name, Desc, Status, Expired]); - undefined -> - emqx_ctl:print("Not Found.~n") + undefined -> emqx_ctl:print("Not Found.~n"); + App -> print_app_info(App) end; mgmt(["update", AppId, Status]) -> @@ -99,10 +98,7 @@ mgmt(["delete", AppId]) -> end; mgmt(["list"]) -> - lists:foreach(fun({AppId, AppSecret, Name, Desc, Status, Expired}) -> - emqx_ctl:print("app_id: ~s, secret: ~s, name: ~s, desc: ~s, status: ~s, expired: ~p~n", - [AppId, AppSecret, Name, Desc, Status, Expired]) - end, emqx_mgmt_auth:list_apps()); + lists:foreach(fun print_app_info/1, emqx_mgmt_auth:list_apps()); mgmt(_) -> emqx_ctl:usage([{"mgmt list", "List Applications"}, @@ -128,10 +124,12 @@ broker([]) -> [emqx_ctl:print("~-10s: ~s~n", [Fun, emqx_sys:Fun()]) || Fun <- Funs]; broker(["stats"]) -> - [emqx_ctl:print("~-30s: ~w~n", [Stat, Val]) || {Stat, Val} <- lists:sort(emqx_stats:getstats())]; + [emqx_ctl:print("~-30s: ~w~n", [Stat, Val]) || + {Stat, Val} <- lists:sort(emqx_stats:getstats())]; broker(["metrics"]) -> - [emqx_ctl:print("~-30s: ~w~n", [Metric, Val]) || {Metric, Val} <- lists:sort(emqx_metrics:all())]; + [emqx_ctl:print("~-30s: ~w~n", [Metric, Val]) || + {Metric, Val} <- lists:sort(emqx_metrics:all())]; broker(_) -> emqx_ctl:usage([{"broker", "Show broker version, uptime and description"}, @@ -256,10 +254,12 @@ subscriptions(["del", ClientId, Topic]) -> end; subscriptions(_) -> - emqx_ctl:usage([{"subscriptions list", "List all subscriptions"}, - {"subscriptions show ", "Show subscriptions of a client"}, - {"subscriptions add ", "Add a static subscription manually"}, - {"subscriptions del ", "Delete a static subscription manually"}]). + emqx_ctl:usage([{"subscriptions list", "List all subscriptions"}, + {"subscriptions show ", "Show subscriptions of a client"}, + {"subscriptions add ", + "Add a static subscription manually"}, + {"subscriptions del ", + "Delete a static subscription manually"}]). if_valid_qos(QoS, Fun) -> try list_to_integer(QoS) of @@ -328,14 +328,20 @@ vm(["memory"]) -> [emqx_ctl:print("memory/~-17s: ~w~n", [Cat, Val]) || {Cat, Val} <- erlang:memory()]; vm(["process"]) -> - [emqx_ctl:print("process/~-16s: ~w~n", [Name, erlang:system_info(Key)]) || {Name, Key} <- [{limit, process_limit}, {count, process_count}]]; + [emqx_ctl:print("process/~-16s: ~w~n", + [Name, erlang:system_info(Key)]) || + {Name, Key} <- [{limit, process_limit}, {count, process_count}]]; vm(["io"]) -> IoInfo = lists:usort(lists:flatten(erlang:system_info(check_io))), - [emqx_ctl:print("io/~-21s: ~w~n", [Key, proplists:get_value(Key, IoInfo)]) || Key <- [max_fds, active_fds]]; + [emqx_ctl:print("io/~-21s: ~w~n", + [Key, proplists:get_value(Key, IoInfo)]) || + Key <- [max_fds, active_fds]]; vm(["ports"]) -> - [emqx_ctl:print("ports/~-16s: ~w~n", [Name, erlang:system_info(Key)]) || {Name, Key} <- [{count, port_count}, {limit, port_limit}]]; + [emqx_ctl:print("ports/~-16s: ~w~n", + [Name, erlang:system_info(Key)]) || + {Name, Key} <- [{count, port_count}, {limit, port_limit}]]; vm(_) -> emqx_ctl:usage([{"vm all", "Show info of Erlang VM"}, @@ -372,8 +378,9 @@ log(["primary-level", Level]) -> emqx_ctl:print("~s~n", [emqx_logger:get_primary_log_level()]); log(["handlers", "list"]) -> - _ = [emqx_ctl:print("LogHandler(id=~s, level=~s, destination=~s, status=~s)~n", [Id, Level, Dst, Status]) - || #{id := Id, level := Level, dst := Dst, status := Status} <- emqx_logger:get_log_handlers()], + _ = [emqx_ctl:print("LogHandler(id=~s, level=~s, destination=~s, status=~s)~n", + [Id, Level, Dst, Status]) || #{id := Id, level := Level, dst := Dst, status := Status} + <- emqx_logger:get_log_handlers()], ok; log(["handlers", "start", HandlerId]) -> @@ -406,15 +413,18 @@ log(_) -> {"log handlers list", "Show log handlers"}, {"log handlers start ", "Start a log handler"}, {"log handlers stop ", "Stop a log handler"}, - {"log handlers set-level ", "Set log level of a log handler"}]). + {"log handlers set-level ", + "Set log level of a log handler"}]). %%-------------------------------------------------------------------- %% @doc Trace Command trace(["list"]) -> - lists:foreach(fun({{Who, Name}, {Level, LogFile}}) -> - emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Who, Name, Level, LogFile]) - end, emqx_tracer:lookup_traces()); + lists:foreach(fun(Trace) -> + #{type := Type, level := Level, dst := Dst} = Trace, + Who = maps:get(Type, Trace), + emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Type, Who, Level, Dst]) + end, emqx_tracer:lookup_traces()); trace(["stop", "client", ClientId]) -> trace_off(clientid, ClientId); @@ -441,8 +451,9 @@ trace(_) -> {"trace start topic [] ", "Traces for a topic"}, {"trace stop topic ", "Stop tracing for a topic"}]). +-dialyzer({nowarn_function, [trace_on/4, trace_off/2]}). trace_on(Who, Name, Level, LogFile) -> - case emqx_tracer:start_trace({Who, iolist_to_binary(Name)}, Level, LogFile) of + case emqx_tracer:start_trace(Who, Name, Level, LogFile) of ok -> emqx_ctl:print("trace ~s ~s successfully~n", [Who, Name]); {error, Error} -> @@ -450,7 +461,7 @@ trace_on(Who, Name, Level, LogFile) -> end. trace_off(Who, Name) -> - case emqx_tracer:stop_trace({Who, iolist_to_binary(Name)}) of + case emqx_tracer:stop_trace(Who, Name) of ok -> emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Name]); {error, Error} -> @@ -472,18 +483,20 @@ listeners([]) -> lists:foreach(fun indent_print/1, Info) end, esockd:listeners()), lists:foreach(fun({Protocol, Opts}) -> - Port = proplists:get_value(port, Opts), - Info = [{listen_on, {string, emqx_listeners:format_listen_on(Port)}}, - {acceptors, maps:get(num_acceptors, proplists:get_value(transport_options, Opts, #{}), 0)}, - {max_conns, proplists:get_value(max_connections, Opts)}, - {current_conn, proplists:get_value(all_connections, Opts)}, - {shutdown_count, []}], - emqx_ctl:print("~s~n", [listener_identifier(Protocol, Port)]), - lists:foreach(fun indent_print/1, Info) - end, ranch:info()); + Port = proplists:get_value(port, Opts), + Acceptors = maps:get(num_acceptors, proplists:get_value(transport_options, Opts, #{}), 0), + Info = [{listen_on, {string, emqx_listeners:format_listen_on(Port)}}, + {acceptors, Acceptors}, + {max_conns, proplists:get_value(max_connections, Opts)}, + {current_conn, proplists:get_value(all_connections, Opts)}, + {shutdown_count, []}], + emqx_ctl:print("~s~n", [listener_identifier(Protocol, Port)]), + lists:foreach(fun indent_print/1, Info) + end, ranch:info()); listeners(["stop", Name = "http" ++ _N | _MaybePort]) -> - %% _MaybePort is to be backward compatible, to stop http listener, there is no need for the port number + %% _MaybePort is to be backward compatible, to stop http listener, + %% there is no need for the port number case minirest:stop_http(list_to_atom(Name)) of ok -> emqx_ctl:print("Stop ~s listener successfully.~n", [Name]); @@ -564,7 +577,8 @@ data(["import", Filename, "--env", Env]) -> {error, unsupported_version} -> emqx_ctl:print("The emqx data import failed: Unsupported version.~n"); {error, Reason} -> - emqx_ctl:print("The emqx data import failed: ~0p while reading ~s.~n", [Reason, Filename]) + emqx_ctl:print("The emqx data import failed: ~0p while reading ~s.~n", + [Reason, Filename]) end; data(_) -> @@ -657,19 +671,23 @@ print({client, {ClientId, ChanPid}}) -> maps:with([created_at], Session)]), InfoKeys = [clientid, username, peername, clean_start, keepalive, expiry_interval, - subscriptions_cnt, inflight_cnt, awaiting_rel_cnt, send_msg, mqueue_len, mqueue_dropped, - connected, created_at, connected_at] ++ case maps:is_key(disconnected_at, Info) of - true -> [disconnected_at]; - false -> [] - end, + subscriptions_cnt, inflight_cnt, awaiting_rel_cnt, + send_msg, mqueue_len, mqueue_dropped, + connected, created_at, connected_at] ++ + case maps:is_key(disconnected_at, Info) of + true -> [disconnected_at]; + false -> [] + end, emqx_ctl:print("Client(~s, username=~s, peername=~s, " - "clean_start=~s, keepalive=~w, session_expiry_interval=~w, " - "subscriptions=~w, inflight=~w, awaiting_rel=~w, delivered_msgs=~w, enqueued_msgs=~w, dropped_msgs=~w, " - "connected=~s, created_at=~w, connected_at=~w" ++ case maps:is_key(disconnected_at, Info) of - true -> ", disconnected_at=~w)~n"; - false -> ")~n" - end, - [format(K, maps:get(K, Info)) || K <- InfoKeys]); + "clean_start=~s, keepalive=~w, session_expiry_interval=~w, " + "subscriptions=~w, inflight=~w, awaiting_rel=~w, " + "delivered_msgs=~w, enqueued_msgs=~w, dropped_msgs=~w, " + "connected=~s, created_at=~w, connected_at=~w" ++ + case maps:is_key(disconnected_at, Info) of + true -> ", disconnected_at=~w)~n"; + false -> ")~n" + end, + [format(K, maps:get(K, Info)) || K <- InfoKeys]); print({emqx_route, #route{topic = Topic, dest = {_, Node}}}) -> emqx_ctl:print("~s -> ~s~n", [Topic, Node]); @@ -721,3 +739,7 @@ restart_http_listener(Scheme, AppName) -> http_mod_name(emqx_management) -> emqx_mgmt_http; http_mod_name(Name) -> Name. + +print_app_info({AppId, AppSecret, Name, Desc, Status, Expired}) -> + emqx_ctl:print("app_id: ~s, secret: ~s, name: ~s, desc: ~s, status: ~s, expired: ~p~n", + [AppId, AppSecret, Name, Desc, Status, Expired]). diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index a96ee7a62..698cbf605 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -17,6 +17,9 @@ -module(emqx_rule_funcs). -include("rule_engine.hrl"). +-elvis([{elvis_style, god_modules, disable}]). +-elvis([{elvis_style, function_naming_convention, disable}]). +-elvis([{elvis_style, macro_names, disable}]). %% IoT Funcs -export([ msgid/0 @@ -438,7 +441,8 @@ subbits(Bits, Len) when is_integer(Len), is_bitstring(Bits) -> subbits(Bits, Start, Len) when is_integer(Start), is_integer(Len), is_bitstring(Bits) -> get_subbits(Bits, Start, Len, <<"integer">>, <<"unsigned">>, <<"big">>). -subbits(Bits, Start, Len, Type, Signedness, Endianness) when is_integer(Start), is_integer(Len), is_bitstring(Bits) -> +subbits(Bits, Start, Len, Type, Signedness, Endianness) + when is_integer(Start), is_integer(Len), is_bitstring(Bits) -> get_subbits(Bits, Start, Len, Type, Signedness, Endianness). get_subbits(Bits, Start, Len, Type, Signedness, Endianness) -> @@ -520,7 +524,7 @@ map(Data) -> emqx_rule_utils:map(Data). bin2hexstr(Bin) when is_binary(Bin) -> - emqx_misc:bin2hexstr_A_F(Bin). + emqx_misc:bin2hexstr_a_f_upper(Bin). hexstr2bin(Str) when is_binary(Str) -> emqx_misc:hexstr2bin(Str). @@ -608,7 +612,8 @@ tokens(S, Separators) -> [list_to_binary(R) || R <- string:lexemes(binary_to_list(S), binary_to_list(Separators))]. tokens(S, Separators, <<"nocrlf">>) -> - [list_to_binary(R) || R <- string:lexemes(binary_to_list(S), binary_to_list(Separators) ++ [$\r,$\n,[$\r,$\n]])]. + [list_to_binary(R) || R <- string:lexemes(binary_to_list(S), + binary_to_list(Separators) ++ [$\r,$\n,[$\r,$\n]])]. concat(S1, S2) when is_binary(S1), is_binary(S2) -> unicode:characters_to_binary([S1, S2], unicode). @@ -646,7 +651,8 @@ replace(SrcStr, P, RepStr) when is_binary(SrcStr), is_binary(P), is_binary(RepSt replace(SrcStr, P, RepStr, <<"all">>) when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> iolist_to_binary(string:replace(SrcStr, P, RepStr, all)); -replace(SrcStr, P, RepStr, <<"trailing">>) when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> +replace(SrcStr, P, RepStr, <<"trailing">>) + when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> iolist_to_binary(string:replace(SrcStr, P, RepStr, trailing)); replace(SrcStr, P, RepStr, <<"leading">>) when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> @@ -662,7 +668,7 @@ regex_replace(SrcStr, RE, RepStr) -> re:replace(SrcStr, RE, RepStr, [global, {return,binary}]). ascii(Char) when is_binary(Char) -> - [FirstC| _] = binary_to_list(Char), + [FirstC | _] = binary_to_list(Char), FirstC. find(S, P) when is_binary(S), is_binary(P) -> @@ -782,7 +788,7 @@ sha256(S) when is_binary(S) -> hash(sha256, S). hash(Type, Data) -> - emqx_misc:bin2hexstr_a_f(crypto:hash(Type, Data)). + emqx_misc:bin2hexstr_a_f_lower(crypto:hash(Type, Data)). %%------------------------------------------------------------------------------ %% Data encode and decode Funcs @@ -875,23 +881,23 @@ time_unit(<<"nanosecond">>) -> nanosecond. %% the function handling to the worker module. %% @end -ifdef(EMQX_ENTERPRISE). -'$handle_undefined_function'(schema_decode, [SchemaId, Data|MoreArgs]) -> +'$handle_undefined_function'(schema_decode, [SchemaId, Data | MoreArgs]) -> emqx_schema_parser:decode(SchemaId, Data, MoreArgs); '$handle_undefined_function'(schema_decode, Args) -> error({args_count_error, {schema_decode, Args}}); -'$handle_undefined_function'(schema_encode, [SchemaId, Term|MoreArgs]) -> +'$handle_undefined_function'(schema_encode, [SchemaId, Term | MoreArgs]) -> emqx_schema_parser:encode(SchemaId, Term, MoreArgs); '$handle_undefined_function'(schema_encode, Args) -> error({args_count_error, {schema_encode, Args}}); -'$handle_undefined_function'(sprintf, [Format|Args]) -> +'$handle_undefined_function'(sprintf, [Format | Args]) -> erlang:apply(fun sprintf_s/2, [Format, Args]); '$handle_undefined_function'(Fun, Args) -> error({sql_function_not_supported, function_literal(Fun, Args)}). -else. -'$handle_undefined_function'(sprintf, [Format|Args]) -> +'$handle_undefined_function'(sprintf, [Format | Args]) -> erlang:apply(fun sprintf_s/2, [Format, Args]); '$handle_undefined_function'(Fun, Args) -> diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index c30e237b7..62cf779ea 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -94,6 +94,8 @@ emqxLoadedPlugins: > emqxLoadedModules: > {emqx_mod_acl_internal, true}. {emqx_mod_presence, true}. + {emqx_mod_trace, false}. + {emqx_mod_st_statistics, false}. {emqx_mod_delayed, false}. {emqx_mod_rewrite, false}. {emqx_mod_subscription, false}. diff --git a/include/emqx_mqtt.hrl b/include/emqx_mqtt.hrl index 5dd9a317c..71c2f25f3 100644 --- a/include/emqx_mqtt.hrl +++ b/include/emqx_mqtt.hrl @@ -542,4 +542,22 @@ -define(SHARE(Group, Topic), emqx_topic:join([<>, Group, Topic])). -define(IS_SHARE(Topic), case Topic of <> -> true; _ -> false end). +-define(TYPE_NAMES, { + 'CONNECT' + , 'CONNACK' + , 'PUBLISH' + , 'PUBACK' + , 'PUBREC' + , 'PUBREL' + , 'PUBCOMP' + , 'SUBSCRIBE' + , 'SUBACK' + , 'UNSUBSCRIBE' + , 'UNSUBACK' + , 'PINGREQ' + , 'PINGRESP' + , 'DISCONNECT' + , 'AUTH' + }). + -endif. diff --git a/lib-ce/emqx_modules/src/emqx_mod_sup.erl b/lib-ce/emqx_modules/src/emqx_mod_sup.erl index 60907f580..109564f65 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_sup.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_sup.erl @@ -77,5 +77,5 @@ init([]) -> assert_started({ok, _Pid}) -> ok; assert_started({ok, _Pid, _Info}) -> ok; -assert_started({error, {already_tarted, _Pid}}) -> ok; +assert_started({error, {already_started, _Pid}}) -> ok; assert_started({error, Reason}) -> erlang:error(Reason). diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace.erl b/lib-ce/emqx_modules/src/emqx_mod_trace.erl new file mode 100644 index 000000000..737ee7e97 --- /dev/null +++ b/lib-ce/emqx_modules/src/emqx_mod_trace.erl @@ -0,0 +1,508 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mod_trace). + +-behaviour(gen_server). +-behaviour(emqx_gen_mod). + + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("kernel/include/file.hrl"). + +-logger_header("[Trace]"). + +%% Mnesia bootstrap +-export([mnesia/1]). + +-boot_mnesia({mnesia, [boot]}). +-copy_mnesia({mnesia, [copy]}). + +-export([ load/1 + , unload/1 + , description/0 + ]). + +-export([ start_link/0 + , list/0 + , list/1 + , get_trace_filename/1 + , create/1 + , delete/1 + , clear/0 + , update/2 + ]). + +-export([ format/1 + , zip_dir/0 + , trace_dir/0 + , trace_file/1 + , delete_files_after_send/2 + ]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). + +-define(TRACE, ?MODULE). +-define(PACKETS, tuple_to_list(?TYPE_NAMES)). +-define(MAX_SIZE, 30). + +-ifdef(TEST). +-export([log_file/2]). +-endif. + +-record(?TRACE, + { name :: binary() | undefined | '_' + , type :: clientid | topic | undefined | '_' + , topic :: emqx_types:topic() | undefined | '_' + , clientid :: emqx_types:clientid() | undefined | '_' + , packets = [] :: list() | '_' + , enable = true :: boolean() | '_' + , start_at :: integer() | undefined | binary() | '_' + , end_at :: integer() | undefined | binary() | '_' + , log_size = #{} :: map() | '_' + }). + +mnesia(boot) -> + ok = ekka_mnesia:create_table(?TRACE, [ + {type, set}, + {disc_copies, [node()]}, + {record_name, ?TRACE}, + {attributes, record_info(fields, ?TRACE)}]); +mnesia(copy) -> + ok = ekka_mnesia:copy_table(?TRACE, disc_copies). + +description() -> + "EMQ X Trace Module". + +-spec load(any()) -> ok. +load(_Env) -> + emqx_mod_sup:start_child(?MODULE, worker). + +-spec unload(any()) -> ok. +unload(_Env) -> + _ = emqx_mod_sup:stop_child(?MODULE), + stop_all_trace_handler(). + +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec list() -> [tuple()]. +list() -> + ets:match_object(?TRACE, #?TRACE{_ = '_'}). + +-spec list(boolean()) -> [tuple()]. +list(Enable) -> + ets:match_object(?TRACE, #?TRACE{enable = Enable, _ = '_'}). + +-spec create([{Key :: binary(), Value :: binary()}]) -> + ok | {error, {duplicate_condition, iodata()} | {already_existed, iodata()} | iodata()}. +create(Trace) -> + case mnesia:table_info(?TRACE, size) < ?MAX_SIZE of + true -> + case to_trace(Trace) of + {ok, TraceRec} -> create_new_trace(TraceRec); + {error, Reason} -> {error, Reason} + end; + false -> + {error, """The number of traces created has reached the maximum, +please delete the useless ones first"""} + end. + +-spec delete(Name :: binary()) -> ok | {error, not_found}. +delete(Name) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name) of + [_] -> mnesia:delete(?TRACE, Name, write); + [] -> mnesia:abort(not_found) + end + end, + transaction(Tran). + +-spec clear() -> ok | {error, Reason :: term()}. +clear() -> + case mnesia:clear_table(?TRACE) of + {atomic, ok} -> ok; + {aborted, Reason} -> {error, Reason} + end. + +-spec update(Name :: binary(), Enable :: boolean()) -> + ok | {error, not_found | finished}. +update(Name, Enable) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name) of + [] -> mnesia:abort(not_found); + [#?TRACE{enable = Enable}] -> ok; + [Rec] -> + case erlang:system_time(second) >= Rec#?TRACE.end_at of + false -> mnesia:write(?TRACE, Rec#?TRACE{enable = Enable}, write); + true -> mnesia:abort(finished) + end + end + end, + transaction(Tran). + +-spec get_trace_filename(Name :: binary()) -> + {ok, FileName :: string()} | {error, not_found}. +get_trace_filename(Name) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name, read) of + [] -> mnesia:abort(not_found); + [#?TRACE{start_at = Start}] -> {ok, filename(Name, Start)} + end end, + transaction(Tran). + +-spec trace_file(File :: list()) -> + {ok, Node :: list(), Binary :: binary()} | + {error, Node :: list(), Reason :: term()}. +trace_file(File) -> + FileName = filename:join(trace_dir(), File), + Node = atom_to_list(node()), + case file:read_file(FileName) of + {ok, Bin} -> {ok, Node, Bin}; + {error, Reason} -> {error, Node, Reason} + end. + +delete_files_after_send(TraceLog, Zips) -> + gen_server:cast(?MODULE, {delete_tag, self(), [TraceLog | Zips]}). + +-spec format(list(#?TRACE{})) -> list(map()). +format(Traces) -> + Fields = record_info(fields, ?TRACE), + lists:map(fun(Trace0 = #?TRACE{start_at = StartAt, end_at = EndAt}) -> + Trace = Trace0#?TRACE{ + start_at = list_to_binary(calendar:system_time_to_rfc3339(StartAt)), + end_at = list_to_binary(calendar:system_time_to_rfc3339(EndAt)) + }, + [_ | Values] = tuple_to_list(Trace), + maps:from_list(lists:zip(Fields, Values)) + end, Traces). + +init([]) -> + erlang:process_flag(trap_exit, true), + OriginLogLevel = emqx_logger:get_primary_log_level(), + ok = filelib:ensure_dir(trace_dir()), + ok = filelib:ensure_dir(zip_dir()), + {ok, _} = mnesia:subscribe({table, ?TRACE, simple}), + Traces = get_enable_trace(), + ok = update_log_primary_level(Traces, OriginLogLevel), + TRef = update_trace(Traces), + {ok, #{timer => TRef, monitors => #{}, primary_log_level => OriginLogLevel}}. + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ok, State}. + +handle_cast({delete_tag, Pid, Files}, State = #{monitors := Monitors}) -> + erlang:monitor(process, Pid), + {noreply, State#{monitors => Monitors#{Pid => Files}}}; +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitors}) -> + case maps:take(Pid, Monitors) of + error -> {noreply, State}; + {Files, NewMonitors} -> + lists:foreach(fun(F) -> file:delete(F) end, Files), + {noreply, State#{monitors => NewMonitors}} + end; +handle_info({timeout, TRef, update_trace}, + #{timer := TRef, primary_log_level := OriginLogLevel} = State) -> + Traces = get_enable_trace(), + ok = update_log_primary_level(Traces, OriginLogLevel), + NextTRef = update_trace(Traces), + {noreply, State#{timer => NextTRef}}; + +handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) -> + emqx_misc:cancel_timer(TRef), + handle_info({timeout, TRef, update_trace}, State); + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, #{timer := TRef, primary_log_level := OriginLogLevel}) -> + ok = set_log_primary_level(OriginLogLevel), + _ = mnesia:unsubscribe({table, ?TRACE, simple}), + emqx_misc:cancel_timer(TRef), + stop_all_trace_handler(), + _ = file:del_dir_r(zip_dir()), + ok. + +code_change(_, State, _Extra) -> + {ok, State}. + +create_new_trace(Trace) -> + Tran = fun() -> + case mnesia:read(?TRACE, Trace#?TRACE.name) of + [] -> + #?TRACE{start_at = StartAt, topic = Topic, + clientid = ClientId, packets = Packets} = Trace, + Match = #?TRACE{_ = '_', start_at = StartAt, topic = Topic, + clientid = ClientId, packets = Packets}, + case mnesia:match_object(?TRACE, Match, read) of + [] -> mnesia:write(?TRACE, Trace, write); + [#?TRACE{name = Name}] -> mnesia:abort({duplicate_condition, Name}) + end; + [#?TRACE{name = Name}] -> mnesia:abort({already_existed, Name}) + end + end, + transaction(Tran). + +update_trace(Traces) -> + Now = erlang:system_time(second), + {_Waiting, Running, Finished} = classify_by_time(Traces, Now), + disable_finished(Finished), + Started = already_running(), + {NeedRunning, AllStarted} = start_trace(Running, Started), + NeedStop = AllStarted -- NeedRunning, + ok = stop_trace(NeedStop, Started), + clean_stale_trace_files(), + NextTime = find_closest_time(Traces, Now), + emqx_misc:start_timer(NextTime, update_trace). + +stop_all_trace_handler() -> + lists:foreach(fun(#{type := Type, name := Name} = Trace) -> + _ = emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name) + end + , already_running()). + +already_running() -> + emqx_tracer:lookup_traces(). + +get_enable_trace() -> + {atomic, Traces} = + mnesia:transaction(fun() -> + mnesia:match_object(?TRACE, #?TRACE{enable = true, _ = '_'}, read) + end), + Traces. + +find_closest_time(Traces, Now) -> + Sec = + lists:foldl( + fun(#?TRACE{start_at = Start, end_at = End}, Closest) + when Start >= Now andalso Now < End -> %% running + min(End - Now, Closest); + (#?TRACE{start_at = Start}, Closest) when Start < Now -> %% waiting + min(Now - Start, Closest); + (_, Closest) -> Closest %% finished + end, 60 * 15, Traces), + timer:seconds(Sec). + +disable_finished([]) -> ok; +disable_finished(Traces) -> + NameWithLogSize = + lists:map(fun(#?TRACE{name = Name, start_at = StartAt}) -> + FileSize = filelib:file_size(log_file(Name, StartAt)), + {Name, FileSize} end, Traces), + transaction(fun() -> + lists:map(fun({Name, LogSize}) -> + case mnesia:read(?TRACE, Name, write) of + [] -> ok; + [Trace = #?TRACE{log_size = Logs}] -> + mnesia:write(?TRACE, Trace#?TRACE{enable = false, + log_size = Logs#{node() => LogSize}}, write) + end end, NameWithLogSize) + end). + +start_trace(Traces, Started0) -> + Started = lists:map(fun(#{name := Name}) -> Name end, Started0), + lists:foldl(fun(#?TRACE{name = Name} = Trace, {Running, StartedAcc}) -> + case lists:member(Name, StartedAcc) of + true -> {[Name | Running], StartedAcc}; + false -> + case start_trace(Trace) of + ok -> {[Name | Running], [Name | StartedAcc]}; + Error -> + ?LOG(error, "(~p)start trace failed by:~p", [Name, Error]), + {[Name | Running], StartedAcc} + end + end + end, {[], Started}, Traces). + +start_trace(Trace) -> + #?TRACE{name = Name + , type = Type + , clientid = ClientId + , topic = Topic + , packets = Packets + , start_at = Start + } = Trace, + Who0 = #{name => Name, labels => Packets}, + Who = + case Type of + topic -> Who0#{type => topic, topic => Topic}; + clientid -> Who0#{type => clientid, clientid => ClientId} + end, + case emqx_tracer:start_trace(Who, debug, log_file(Name, Start)) of + ok -> ok; + {error, {already_exist, _}} -> ok; + {error, Reason} -> {error, Reason} + end. + +stop_trace(Finished, Started) -> + lists:foreach(fun(#{name := Name, type := Type} = Trace) -> + case lists:member(Name, Finished) of + true -> emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name); + false -> ok + end + end, Started). + +clean_stale_trace_files() -> + TraceDir = trace_dir(), + case file:list_dir(TraceDir) of + {ok, AllFiles} when AllFiles =/= ["zip"] -> + FileFun = fun(#?TRACE{name = Name, start_at = StartAt}) -> filename(Name, StartAt) end, + KeepFiles = lists:map(FileFun, list()), + case AllFiles -- ["zip" | KeepFiles] of + [] -> ok; + DeleteFiles -> + DelFun = fun(F) -> file:delete(filename:join(TraceDir, F)) end, + lists:foreach(DelFun, DeleteFiles) + end; + _ -> ok + end. + +classify_by_time(Traces, Now) -> + classify_by_time(Traces, Now, [], [], []). + +classify_by_time([], _Now, Wait, Run, Finish) -> {Wait, Run, Finish}; +classify_by_time([Trace = #?TRACE{start_at = Start} | Traces], + Now, Wait, Run, Finish) when Start > Now -> + classify_by_time(Traces, Now, [Trace | Wait], Run, Finish); +classify_by_time([Trace = #?TRACE{end_at = End} | Traces], + Now, Wait, Run, Finish) when End =< Now -> + classify_by_time(Traces, Now, Wait, Run, [Trace | Finish]); +classify_by_time([Trace | Traces], Now, Wait, Run, Finish) -> + classify_by_time(Traces, Now, Wait, [Trace | Run], Finish). + +to_trace(TraceList) -> + case to_trace(TraceList, #?TRACE{}) of + {error, Reason} -> {error, Reason}; + {ok, #?TRACE{name = undefined}} -> + {error, "name required"}; + {ok, #?TRACE{type = undefined}} -> + {error, "type required"}; + {ok, #?TRACE{topic = undefined, clientid = undefined}} -> + {error, "topic/clientid cannot be both empty"}; + {ok, Trace} -> + case fill_default(Trace) of + #?TRACE{start_at = Start, end_at = End} when End =< Start -> + {error, "failed by start_at >= end_at"}; + Trace1 -> {ok, Trace1} + end + end. + +fill_default(Trace = #?TRACE{start_at = undefined}) -> + fill_default(Trace#?TRACE{start_at = erlang:system_time(second)}); +fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) -> + fill_default(Trace#?TRACE{end_at = StartAt + 10 * 60}); +fill_default(Trace) -> Trace. + +to_trace([], Rec) -> {ok, Rec}; +to_trace([{<<"name">>, Name} | Trace], Rec) -> + case binary:match(Name, [<<"/">>], []) of + nomatch -> to_trace(Trace, Rec#?TRACE{name = Name}); + _ -> {error, "name cannot contain /"} + end; +to_trace([{<<"type">>, Type} | Trace], Rec) -> + case lists:member(Type, [<<"clientid">>, <<"topic">>]) of + true -> to_trace(Trace, Rec#?TRACE{type = binary_to_existing_atom(Type)}); + false -> {error, "incorrect type: only support clientid/topic"} + end; +to_trace([{<<"topic">>, Topic} | Trace], Rec) -> + case validate_topic(Topic) of + ok -> to_trace(Trace, Rec#?TRACE{topic = Topic}); + {error, Reason} -> {error, Reason} + end; +to_trace([{<<"start_at">>, StartAt} | Trace], Rec) -> + case to_system_second(StartAt) of + {ok, Sec} -> to_trace(Trace, Rec#?TRACE{start_at = Sec}); + {error, Reason} -> {error, Reason} + end; +to_trace([{<<"end_at">>, EndAt} | Trace], Rec) -> + Now = erlang:system_time(second), + case to_system_second(EndAt) of + {ok, Sec} when Sec > Now -> + to_trace(Trace, Rec#?TRACE{end_at = Sec}); + {ok, _Sec} -> + {error, "end_at time has already passed"}; + {error, Reason} -> + {error, Reason} + end; +to_trace([{<<"clientid">>, ClientId} | Trace], Rec) -> + to_trace(Trace, Rec#?TRACE{clientid = ClientId}); +to_trace([{<<"packets">>, PacketList} | Trace], Rec) -> + case to_packets(PacketList) of + {ok, Packets} -> to_trace(Trace, Rec#?TRACE{packets = Packets}); + {error, Reason} -> {error, io_lib:format("unsupport packets: ~p", [Reason])} + end; +to_trace([Unknown | _Trace], _Rec) -> {error, io_lib:format("unknown field: ~p", [Unknown])}. + +validate_topic(TopicName) -> + try emqx_topic:validate(name, TopicName) of + true -> ok + catch + error:Error -> + {error, io_lib:format("~s invalid by ~p", [TopicName, Error])} + end. + +to_system_second(At) -> + try + Sec = calendar:rfc3339_to_system_time(binary_to_list(At), [{unit, second}]), + {ok, Sec} + catch error: {badmatch, _} -> + {error, ["The rfc3339 specification not satisfied: ", At]} + end. + +to_packets(Packets) when is_list(Packets) -> + AtomTypes = lists:map(fun(Type) -> binary_to_existing_atom(Type) end, Packets), + case lists:filter(fun(T) -> not lists:member(T, ?PACKETS) end, AtomTypes) of + [] -> {ok, AtomTypes}; + InvalidE -> {error, InvalidE} + end; +to_packets(Packets) -> {error, Packets}. + +zip_dir() -> + trace_dir() ++ "zip/". + +trace_dir() -> + filename:join(emqx:get_env(data_dir), "trace") ++ "/". + +log_file(Name, Start) -> + filename:join(trace_dir(), filename(Name, Start)). + +filename(Name, Start) -> + [Time, _] = string:split(calendar:system_time_to_rfc3339(Start), "T", leading), + lists:flatten(["trace_", binary_to_list(Name), "_", Time, ".log"]). + +transaction(Tran) -> + case mnesia:transaction(Tran) of + {atomic, Res} -> Res; + {aborted, Reason} -> {error, Reason} + end. + +update_log_primary_level([], OriginLevel) -> set_log_primary_level(OriginLevel); +update_log_primary_level(_, _) -> set_log_primary_level(debug). + +set_log_primary_level(NewLevel) -> + case NewLevel =/= emqx_logger:get_primary_log_level() of + true -> emqx_logger:set_primary_log_level(NewLevel); + false -> ok + end. diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl new file mode 100644 index 000000000..6d2813b96 --- /dev/null +++ b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl @@ -0,0 +1,191 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mod_trace_api). +-include_lib("emqx/include/logger.hrl"). + +%% API +-export([ list_trace/2 + , create_trace/2 + , update_trace/2 + , delete_trace/2 + , clear_traces/2 + , download_zip_log/2 + , stream_log_file/2 +]). +-export([read_trace_file/3]). + +-define(TO_BIN(_B_), iolist_to_binary(_B_)). +-define(RETURN_NOT_FOUND(N), return({error, 'NOT_FOUND', ?TO_BIN([N, "NOT FOUND"])})). + +-import(minirest, [return/1]). + +-rest_api(#{name => list_trace, + method => 'GET', + path => "/trace/", + func => list_trace, + descr => "list all traces"}). + +-rest_api(#{name => create_trace, + method => 'POST', + path => "/trace/", + func => create_trace, + descr => "create trace"}). + +-rest_api(#{name => delete_trace, + method => 'DELETE', + path => "/trace/:bin:name", + func => delete_trace, + descr => "delete trace"}). + +-rest_api(#{name => clear_trace, + method => 'DELETE', + path => "/trace/", + func => clear_traces, + descr => "clear all traces"}). + +-rest_api(#{name => update_trace, + method => 'PUT', + path => "/trace/:bin:name/:atom:operation", + func => update_trace, + descr => "diable/enable trace"}). + +-rest_api(#{name => download_zip_log, + method => 'GET', + path => "/trace/:bin:name/download", + func => download_zip_log, + descr => "download trace's log"}). + +-rest_api(#{name => stream_log_file, + method => 'GET', + path => "/trace/:bin:name/log", + func => stream_log_file, + descr => "download trace's log"}). + +list_trace(_, Params) -> + List = + case Params of + [{<<"enable">>, Enable}] -> emqx_mod_trace:list(Enable); + _ -> emqx_mod_trace:list() + end, + return({ok, emqx_mod_trace:format(List)}). + +create_trace(_, Param) -> + case emqx_mod_trace:create(Param) of + ok -> return(ok); + {error, {already_existed, Name}} -> + return({error, 'ALREADY_EXISTED', ?TO_BIN([Name, "Already Exists"])}); + {error, {duplicate_condition, Name}} -> + return({error, 'DUPLICATE_CONDITION', ?TO_BIN([Name, "Duplication Condition"])}); + {error, Reason} -> + return({error, 'INCORRECT_PARAMS', ?TO_BIN(Reason)}) + end. + +delete_trace(#{name := Name}, _Param) -> + case emqx_mod_trace:delete(Name) of + ok -> return(ok); + {error, not_found} -> ?RETURN_NOT_FOUND(Name) + end. + +clear_traces(_, _) -> + return(emqx_mod_trace:clear()). + +update_trace(#{name := Name, operation := Operation}, _Param) -> + Enable = case Operation of disable -> false; enable -> true end, + case emqx_mod_trace:update(Name, Enable) of + ok -> return({ok, #{enable => Enable, name => Name}}); + {error, not_found} -> ?RETURN_NOT_FOUND(Name) + end. + +%% if HTTP request headers include accept-encoding: gzip and file size > 300 bytes. +%% cowboy_compress_h will auto encode gzip format. +download_zip_log(#{name := Name}, _Param) -> + case emqx_mod_trace:get_trace_filename(Name) of + {ok, TraceLog} -> + TraceFiles = collect_trace_file(TraceLog), + ZipDir = emqx_mod_trace:zip_dir(), + Zips = group_trace_file(ZipDir, TraceLog, TraceFiles), + ZipFileName = ZipDir ++ TraceLog, + {ok, ZipFile} = zip:zip(ZipFileName, Zips), + emqx_mod_trace:delete_files_after_send(ZipFileName, Zips), + {ok, #{}, {sendfile, 0, filelib:file_size(ZipFile), ZipFile}}; + {error, Reason} -> + return({error, Reason}) + end. + +group_trace_file(ZipDir, TraceLog, TraceFiles) -> + lists:foldl(fun(Res, Acc) -> + case Res of + {ok, Node, Bin} -> + ZipName = ZipDir ++ Node ++ "-" ++ TraceLog, + ok = file:write_file(ZipName, Bin), + [ZipName | Acc]; + {error, Node, Reason} -> + ?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]), + Acc + end + end, [], TraceFiles). + +collect_trace_file(TraceLog) -> + Nodes = ekka_mnesia:running_nodes(), + {Files, BadNodes} = rpc:multicall(Nodes, emqx_mod_trace, trace_file, [TraceLog], 60000), + BadNodes =/= [] andalso ?LOG(error, "download log rpc failed on ~p", [BadNodes]), + Files. + +%% _page as position and _limit as bytes for front-end reusing components +stream_log_file(#{name := Name}, Params) -> + Node0 = proplists:get_value(<<"node">>, Params, atom_to_binary(node())), + Position0 = proplists:get_value(<<"_page">>, Params, <<"0">>), + Bytes0 = proplists:get_value(<<"_limit">>, Params, <<"500">>), + Node = binary_to_existing_atom(Node0), + Position = binary_to_integer(Position0), + Bytes = binary_to_integer(Bytes0), + case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of + {ok, Bin} -> + Meta = #{<<"page">> => Position + byte_size(Bin), <<"limit">> => Bytes}, + return({ok, #{meta => Meta, items => Bin}}); + eof -> + Meta = #{<<"page">> => Position, <<"limit">> => Bytes}, + return({ok, #{meta => Meta, items => <<"">>}}); + {error, Reason} -> + logger:log(error, "read_file_failed by ~p", [{Name, Reason, Position, Bytes}]), + return({error, Reason}) + end. + +%% this is an rpc call for stream_log_file/2 +read_trace_file(Name, Position, Limit) -> + TraceDir = emqx_mod_trace:trace_dir(), + {ok, AllFiles} = file:list_dir(TraceDir), + TracePrefix = "trace_" ++ binary_to_list(Name) ++ "_", + Filter = fun(FileName) -> nomatch =/= string:prefix(FileName, TracePrefix) end, + case lists:filter(Filter, AllFiles) of + [TraceFile] -> + TracePath = filename:join([TraceDir, TraceFile]), + read_file(TracePath, Position, Limit); + [] -> {error, not_found} + end. + +read_file(Path, Offset, Bytes) -> + {ok, IoDevice} = file:open(Path, [read, raw, binary]), + try + _ = case Offset of + 0 -> ok; + _ -> file:position(IoDevice, {bof, Offset}) + end, + file:read(IoDevice, Bytes) + after + file:close(IoDevice) + end. diff --git a/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl index 997eff1c2..e6a9f6c16 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl @@ -62,7 +62,7 @@ t_mod_rewrite(_Config) -> timer:sleep(100), ?assertEqual([], emqx_broker:subscriptions(<<"rewrite_client">>)), %% Pub Rules - {ok, _Props, _} = emqtt:subscribe(C, [{Topic, ?QOS_1} || Topic <- PubDestTopics]), + {ok, _Props1, _} = emqtt:subscribe(C, [{Topic, ?QOS_1} || Topic <- PubDestTopics]), RecvTopics2 = [begin ok = emqtt:publish(C, Topic, <<"payload">>), {ok, #{topic := RecvTopic}} = receive_publish(100), diff --git a/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl index 59d0ffde2..7c666ea9a 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl @@ -41,9 +41,8 @@ t_start_child(_) -> modules => [Mod]}, ok = emqx_mod_sup:start_child(Mod, worker), - ?assertError({already_started, _}, emqx_mod_sup:start_child(Spec)), + ?assertEqual(ok, emqx_mod_sup:start_child(Spec)), ok = emqx_mod_sup:stop_child(Mod), {error, not_found} = emqx_mod_sup:stop_child(Mod), ok. - diff --git a/lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl new file mode 100644 index 000000000..c643c7908 --- /dev/null +++ b/lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl @@ -0,0 +1,478 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mod_trace_SUITE). + +%% API +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-define(HOST, "http://127.0.0.1:18083/"). +-define(API_VERSION, "v4"). +-define(BASE_PATH, "api"). + +-record(emqx_mod_trace, { + name, + type, + topic, + clientid, + packets = [], + enable = true, + start_at, + end_at, + log_size = #{} + }). + +-define(PACKETS, ['CONNECT', 'CONNACK', 'PUBLISH', 'PUBACK', 'PUBREC', 'PUBREL' + , 'PUBCOMP', 'SUBSCRIBE', 'SUBACK', 'UNSUBSCRIBE', 'UNSUBACK' + , 'PINGREQ', 'PINGRESP', 'DISCONNECT', 'AUTH']). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_modules, emqx_dashboard]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_modules, emqx_dashboard]). + +t_base_create_delete(_Config) -> + ok = emqx_mod_trace:clear(), + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + End = to_rfc3339(Now + 30 * 60), + Name = <<"name1">>, + ClientId = <<"test-device">>, + Packets = [atom_to_binary(E) || E <- ?PACKETS], + Trace = [ + {<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, + {<<"clientid">>, ClientId}, + {<<"packets">>, Packets}, + {<<"start_at">>, Start}, + {<<"end_at">>, End} + ], + AnotherTrace = lists:keyreplace(<<"name">>, 1, Trace, {<<"name">>, <<"AnotherName">>}), + ok = emqx_mod_trace:create(Trace), + ?assertEqual({error, {already_existed, Name}}, emqx_mod_trace:create(Trace)), + ?assertEqual({error, {duplicate_condition, Name}}, emqx_mod_trace:create(AnotherTrace)), + [TraceRec] = emqx_mod_trace:list(), + Expect = #emqx_mod_trace{ + name = Name, + type = clientid, + topic = undefined, + clientid = ClientId, + packets = ?PACKETS, + start_at = Now, + end_at = Now + 30 * 60 + }, + ?assertEqual(Expect, TraceRec), + ExpectFormat = [ + #{ + clientid => <<"test-device">>, + enable => true, + type => clientid, + packets => ?PACKETS, + name => <<"name1">>, + start_at => Start, + end_at => End, + log_size => #{}, + topic => undefined + } + ], + ?assertEqual(ExpectFormat, emqx_mod_trace:format([TraceRec])), + ?assertEqual(ok, emqx_mod_trace:delete(Name)), + ?assertEqual({error, not_found}, emqx_mod_trace:delete(Name)), + ?assertEqual([], emqx_mod_trace:list()), + ok. + +t_create_size_max(_Config) -> + emqx_mod_trace:clear(), + lists:map(fun(Seq) -> + Name = list_to_binary("name" ++ integer_to_list(Seq)), + Trace = [{<<"name">>, Name}, {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"topic">>, list_to_binary("/x/y/" ++ integer_to_list(Seq))}], + ok = emqx_mod_trace:create(Trace) + end, lists:seq(1, 30)), + Trace31 = [{<<"name">>, <<"name31">>}, {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/31">>}], + {error, _} = emqx_mod_trace:create(Trace31), + ok = emqx_mod_trace:delete(<<"name30">>), + ok = emqx_mod_trace:create(Trace31), + ?assertEqual(30, erlang:length(emqx_mod_trace:list())), + ok. + +t_create_failed(_Config) -> + ok = emqx_mod_trace:clear(), + UnknownField = [{<<"unknown">>, 12}], + {error, Reason1} = emqx_mod_trace:create(UnknownField), + ?assertEqual(<<"unknown field: {<<\"unknown\">>,12}">>, iolist_to_binary(Reason1)), + + InvalidTopic = [{<<"topic">>, "#/#//"}], + {error, Reason2} = emqx_mod_trace:create(InvalidTopic), + ?assertEqual(<<"#/#// invalid by function_clause">>, iolist_to_binary(Reason2)), + + InvalidStart = [{<<"start_at">>, <<"2021-12-3:12">>}], + {error, Reason3} = emqx_mod_trace:create(InvalidStart), + ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, + iolist_to_binary(Reason3)), + + InvalidEnd = [{<<"end_at">>, <<"2021-12-3:12">>}], + {error, Reason4} = emqx_mod_trace:create(InvalidEnd), + ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, + iolist_to_binary(Reason4)), + + InvalidPackets = [{<<"packets">>, [<<"publish">>]}], + {error, Reason5} = emqx_mod_trace:create(InvalidPackets), + ?assertEqual(<<"unsupport packets: [publish]">>, iolist_to_binary(Reason5)), + + InvalidPackets2 = [{<<"packets">>, <<"publish">>}], + {error, Reason6} = emqx_mod_trace:create(InvalidPackets2), + ?assertEqual(<<"unsupport packets: <<\"publish\">>">>, iolist_to_binary(Reason6)), + + {error, Reason7} = emqx_mod_trace:create([{<<"name">>, <<"test">>}, {<<"type">>, <<"clientid">>}]), + ?assertEqual(<<"topic/clientid cannot be both empty">>, iolist_to_binary(Reason7)), + + InvalidPackets4 = [{<<"name">>, <<"/test">>}, {<<"clientid">>, <<"t">>}, {<<"type">>, <<"clientid">>}], + {error, Reason9} = emqx_mod_trace:create(InvalidPackets4), + ?assertEqual(<<"name cannot contain /">>, iolist_to_binary(Reason9)), + + ?assertEqual({error, "type required"}, emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, + {<<"packets">>, []}, {<<"clientid">>, <<"good">>}])), + + ?assertEqual({error, "incorrect type: only support clientid/topic"}, + emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, + {<<"packets">>, []}, {<<"clientid">>, <<"good">>}, {<<"type">>, <<"typeerror">> }])), + ok. + +t_create_default(_Config) -> + ok = emqx_mod_trace:clear(), + {error, "name required"} = emqx_mod_trace:create([]), + ok = emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, + {<<"type">>, <<"clientid">>}, {<<"packets">>, []}, {<<"clientid">>, <<"good">>}]), + [#emqx_mod_trace{packets = Packets}] = emqx_mod_trace:list(), + ?assertEqual([], Packets), + ok = emqx_mod_trace:clear(), + Trace = [ + {<<"name">>, <<"test-name">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/z">>}, + {<<"start_at">>, <<"2021-10-28T10:54:47+08:00">>}, + {<<"end_at">>, <<"2021-10-27T10:54:47+08:00">>} + ], + {error, "end_at time has already passed"} = emqx_mod_trace:create(Trace), + Now = erlang:system_time(second), + Trace2 = [ + {<<"name">>, <<"test-name">>}, + {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"topic">>, <<"/x/y/z">>}, + {<<"start_at">>, to_rfc3339(Now + 10)}, + {<<"end_at">>, to_rfc3339(Now + 3)} + ], + {error, "failed by start_at >= end_at"} = emqx_mod_trace:create(Trace2), + ok = emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, + {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/z">>}]), + [#emqx_mod_trace{start_at = Start, end_at = End}] = emqx_mod_trace:list(), + ?assertEqual(10 * 60, End - Start), + ?assertEqual(true, Start - erlang:system_time(second) < 5), + ok. + +t_update_enable(_Config) -> + ok = emqx_mod_trace:clear(), + Name = <<"test-name">>, + Now = erlang:system_time(second), + End = list_to_binary(calendar:system_time_to_rfc3339(Now + 2)), + ok = emqx_mod_trace:create([{<<"name">>, Name}, {<<"packets">>, [<<"PUBLISH">>]}, + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/z">>}, {<<"end_at">>, End}]), + [#emqx_mod_trace{enable = Enable}] = emqx_mod_trace:list(), + ?assertEqual(Enable, true), + ok = emqx_mod_trace:update(Name, false), + [#emqx_mod_trace{enable = false}] = emqx_mod_trace:list(), + ok = emqx_mod_trace:update(Name, false), + [#emqx_mod_trace{enable = false}] = emqx_mod_trace:list(), + ok = emqx_mod_trace:update(Name, true), + [#emqx_mod_trace{enable = true}] = emqx_mod_trace:list(), + ok = emqx_mod_trace:update(Name, false), + [#emqx_mod_trace{enable = false}] = emqx_mod_trace:list(), + ?assertEqual({error, not_found}, emqx_mod_trace:update(<<"Name not found">>, true)), + ct:sleep(2100), + ?assertEqual({error, finished}, emqx_mod_trace:update(Name, true)), + ok. + +t_load_state(_Config) -> + emqx_mod_trace:clear(), + ok = emqx_mod_trace:load(test), + Now = erlang:system_time(second), + Running = [{<<"name">>, <<"Running">>}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/1">>}, {<<"start_at">>, to_rfc3339(Now - 1)}, {<<"end_at">>, to_rfc3339(Now + 2)}], + Waiting = [{<<"name">>, <<"Waiting">>}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/2">>}, {<<"start_at">>, to_rfc3339(Now + 3)}, {<<"end_at">>, to_rfc3339(Now + 8)}], + Finished = [{<<"name">>, <<"Finished">>}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/3">>}, {<<"start_at">>, to_rfc3339(Now - 5)}, {<<"end_at">>, to_rfc3339(Now)}], + ok = emqx_mod_trace:create(Running), + ok = emqx_mod_trace:create(Waiting), + {error, "end_at time has already passed"} = emqx_mod_trace:create(Finished), + Traces = emqx_mod_trace:format(emqx_mod_trace:list()), + ?assertEqual(2, erlang:length(Traces)), + Enables = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces), + ExpectEnables = [{<<"Running">>, true}, {<<"Waiting">>, true}], + ?assertEqual(ExpectEnables, lists:sort(Enables)), + ct:sleep(3500), + Traces2 = emqx_mod_trace:format(emqx_mod_trace:list()), + ?assertEqual(2, erlang:length(Traces2)), + Enables2 = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces2), + ExpectEnables2 = [{<<"Running">>, false}, {<<"Waiting">>, true}], + ?assertEqual(ExpectEnables2, lists:sort(Enables2)), + ok = emqx_mod_trace:unload(test), + ok. + +t_client_event(_Config) -> + application:set_env(emqx, allow_anonymous, true), + emqx_mod_trace:clear(), + ClientId = <<"client-test">>, + ok = emqx_mod_trace:load(test), + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + Name = <<"test_client_id_event">>, + ok = emqx_mod_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + ct:sleep(200), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + emqtt:ping(Client), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"1">>, [{qos, 0}]), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"2">>, [{qos, 0}]), + ct:sleep(200), + ok = emqx_mod_trace:create([{<<"name">>, <<"test_topic">>}, + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/test">>}, {<<"start_at">>, Start}]), + ct:sleep(200), + {ok, Bin} = file:read_file(emqx_mod_trace:log_file(Name, Now)), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"3">>, [{qos, 0}]), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"4">>, [{qos, 0}]), + ok = emqtt:disconnect(Client), + ct:sleep(200), + {ok, Bin2} = file:read_file(emqx_mod_trace:log_file(Name, Now)), + {ok, Bin3} = file:read_file(emqx_mod_trace:log_file(<<"test_topic">>, Now)), + ct:pal("Bin ~p Bin2 ~p Bin3 ~p", [byte_size(Bin), byte_size(Bin2), byte_size(Bin3)]), + ?assert(erlang:byte_size(Bin) > 0), + ?assert(erlang:byte_size(Bin) < erlang:byte_size(Bin2)), + ?assert(erlang:byte_size(Bin3) > 0), + ok = emqx_mod_trace:unload(test), + ok. + +t_get_log_filename(_Config) -> + ok = emqx_mod_trace:clear(), + ok = emqx_mod_trace:load(test), + Now = erlang:system_time(second), + Start = calendar:system_time_to_rfc3339(Now), + End = calendar:system_time_to_rfc3339(Now + 2), + Name = <<"name1">>, + ClientId = <<"test-device">>, + Packets = [atom_to_binary(E) || E <- ?PACKETS], + Trace = [ + {<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, + {<<"clientid">>, ClientId}, + {<<"packets">>, Packets}, + {<<"start_at">>, list_to_binary(Start)}, + {<<"end_at">>, list_to_binary(End)} + ], + ok = emqx_mod_trace:create(Trace), + ?assertEqual({error, not_found}, emqx_mod_trace:get_trace_filename(<<"test">>)), + ?assertEqual(ok, element(1, emqx_mod_trace:get_trace_filename(Name))), + ct:sleep(3000), + ?assertEqual(ok, element(1, emqx_mod_trace:get_trace_filename(Name))), + ok = emqx_mod_trace:unload(test), + ok. + +t_trace_file(_Config) -> + FileName = "test.log", + Content = <<"test \n test">>, + TraceDir = emqx_mod_trace:trace_dir(), + File = filename:join(TraceDir, FileName), + ok = file:write_file(File, Content), + {ok, Node, Bin} = emqx_mod_trace:trace_file(FileName), + ?assertEqual(Node, atom_to_list(node())), + ?assertEqual(Content, Bin), + ok = file:delete(File), + ok. + +t_http_test(_Config) -> + emqx_mod_trace:clear(), + emqx_mod_trace:load(test), + Header = auth_header_(), + %% list + {ok, Empty} = request_api(get, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(Empty)), + %% create + ErrorTrace = #{}, + {ok, Error} = request_api(post, api_path("trace"), Header, ErrorTrace), + ?assertEqual(#{<<"message">> => <<"unknown field: {}">>, <<"code">> => <<"INCORRECT_PARAMS">>}, json(Error)), + + Name = <<"test-name">>, + Trace = [ + {<<"name">>, Name}, + {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"topic">>, <<"/x/y/z">>} + ], + + {ok, Create} = request_api(post, api_path("trace"), Header, Trace), + ?assertEqual(#{<<"code">> => 0}, json(Create)), + + {ok, List} = request_api(get, api_path("trace"), Header), + #{<<"code">> := 0, <<"data">> := [Data]} = json(List), + ?assertEqual(Name, maps:get(<<"name">>, Data)), + + %% update + {ok, Update} = request_api(put, api_path("trace/test-name/disable"), Header, #{}), + ?assertEqual(#{<<"code">> => 0, + <<"data">> => #{<<"enable">> => false, + <<"name">> => <<"test-name">>}}, json(Update)), + + {ok, List1} = request_api(get, api_path("trace"), Header), + #{<<"code">> := 0, <<"data">> := [Data1]} = json(List1), + ?assertEqual(false, maps:get(<<"enable">>, Data1)), + + %% delete + {ok, Delete} = request_api(delete, api_path("trace/test-name"), Header), + ?assertEqual(#{<<"code">> => 0}, json(Delete)), + + {ok, DeleteNotFound} = request_api(delete, api_path("trace/test-name"), Header), + ?assertEqual(#{<<"code">> => <<"NOT_FOUND">>, + <<"message">> => <<"test-nameNOT FOUND">>}, json(DeleteNotFound)), + + {ok, List2} = request_api(get, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(List2)), + + %% clear + {ok, Create1} = request_api(post, api_path("trace"), Header, Trace), + ?assertEqual(#{<<"code">> => 0}, json(Create1)), + + {ok, Clear} = request_api(delete, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0}, json(Clear)), + + emqx_mod_trace:unload(test), + ok. + +t_download_log(_Config) -> + emqx_mod_trace:clear(), + emqx_mod_trace:load(test), + ClientId = <<"client-test">>, + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + Name = <<"test_client_id">>, + ok = emqx_mod_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], + ct:sleep(100), + {ok, #{}, {sendfile, 0, ZipFileSize, _ZipFile}} = + emqx_mod_trace_api:download_zip_log(#{name => Name}, []), + ?assert(ZipFileSize > 0), + %% download zip file failed by server_closed occasionally? + %Header = auth_header_(), + %{ok, ZipBin} = request_api(get, api_path("trace/test_client_id/download"), Header), + %{ok, ZipHandler} = zip:zip_open(ZipBin), + %{ok, [ZipName]} = zip:zip_get(ZipHandler), + %?assertNotEqual(nomatch, string:find(ZipName, "test@127.0.0.1")), + %{ok, _} = file:read_file(emqx_mod_trace:log_file(<<"test_client_id">>, Now)), + ok = emqtt:disconnect(Client), + emqx_mod_trace:unload(test), + ok. + +t_stream_log(_Config) -> + application:set_env(emqx, allow_anonymous, true), + emqx_mod_trace:clear(), + emqx_mod_trace:load(test), + ClientId = <<"client-stream">>, + Now = erlang:system_time(second), + Name = <<"test_stream_log">>, + Start = to_rfc3339(Now - 10), + ok = emqx_mod_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + ct:sleep(200), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], + emqtt:publish(Client, <<"/good">>, #{}, <<"ghood1">>, [{qos, 0}]), + emqtt:publish(Client, <<"/good">>, #{}, <<"ghood2">>, [{qos, 0}]), + ok = emqtt:disconnect(Client), + ct:sleep(200), + File = emqx_mod_trace:log_file(Name, Now), + ct:pal("FileName: ~p", [File]), + {ok, FileBin} = file:read_file(File), + ct:pal("FileBin: ~p ~s", [byte_size(FileBin), FileBin]), + Header = auth_header_(), + {ok, Binary} = request_api(get, api_path("trace/test_stream_log/log?_limit=10"), Header), + #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta, <<"items">> := Bin}} = json(Binary), + ?assertEqual(10, byte_size(Bin)), + ?assertEqual(#{<<"page">> => 10, <<"limit">> => 10}, Meta), + {ok, Binary1} = request_api(get, api_path("trace/test_stream_log/log?_page=20&_limit=10"), Header), + #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta1, <<"items">> := Bin1}} = json(Binary1), + ?assertEqual(#{<<"page">> => 30, <<"limit">> => 10}, Meta1), + ?assertEqual(10, byte_size(Bin1)), + emqx_mod_trace:unload(test), + ok. + +to_rfc3339(Second) -> + list_to_binary(calendar:system_time_to_rfc3339(Second)). + +auth_header_() -> + auth_header_("admin", "public"). + +auth_header_(User, Pass) -> + Encoded = base64:encode_to_string(lists:append([User, ":", Pass])), + {"Authorization", "Basic " ++ Encoded}. + +request_api(Method, Url, Auth) -> do_request_api(Method, {Url, [Auth]}). + +request_api(Method, Url, Auth, Body) -> + Request = {Url, [Auth], "application/json", emqx_json:encode(Body)}, + do_request_api(Method, Request). + +do_request_api(Method, Request) -> + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], [{body_format, binary}]) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {error,{shutdown, server_closed}} -> + {error, server_closed}; + {ok, {{"HTTP/1.1", Code, _}, _Headers, Return} } + when Code =:= 200 orelse Code =:= 201 -> + {ok, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +api_path(Path) -> + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION, Path]). + +json(Data) -> + {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), Jsx. diff --git a/priv/emqx.schema b/priv/emqx.schema index febd4cb9e..69f3acc41 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -2259,6 +2259,7 @@ end}. [{emqx_mod_rewrite, Rewrites()}], [{emqx_mod_topic_metrics, []}], [{emqx_mod_delayed, []}], + [{emqx_mod_trace, []}], [{emqx_mod_st_statistics, SlowTopic()}], [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] ]) diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index d3ad128bb..14617e2e1 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -82,7 +82,7 @@ -define(SUBSCRIPTION, emqx_subscription). %% Guards --define(is_subid(Id), (is_binary(Id) orelse is_atom(Id))). +-define(IS_SUBID(Id), (is_binary(Id) orelse is_atom(Id))). -spec(start_link(atom(), pos_integer()) -> startlink_ret()). start_link(Pool, Id) -> @@ -118,14 +118,15 @@ subscribe(Topic) when is_binary(Topic) -> subscribe(Topic, undefined). -spec(subscribe(emqx_topic:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok). -subscribe(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) -> +subscribe(Topic, SubId) when is_binary(Topic), ?IS_SUBID(SubId) -> subscribe(Topic, SubId, ?DEFAULT_SUBOPTS); subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) -> subscribe(Topic, undefined, SubOpts). -spec(subscribe(emqx_topic:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok). -subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?is_subid(SubId), is_map(SubOpts0) -> +subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) -> SubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts0), + _ = emqx_tracer:trace_subscribe(Topic, SubId, SubOpts), case ets:member(?SUBOPTION, {SubPid = self(), Topic}) of false -> %% New ok = emqx_broker_helper:register_sub(SubPid, SubId), @@ -171,6 +172,7 @@ unsubscribe(Topic) when is_binary(Topic) -> SubPid = self(), case ets:lookup(?SUBOPTION, {SubPid, Topic}) of [{_, SubOpts}] -> + emqx_tracer:trace_unsubscribe(Topic, SubOpts), _ = emqx_broker_helper:reclaim_seq(Topic), do_unsubscribe(Topic, SubPid, SubOpts); [] -> ok @@ -183,13 +185,7 @@ do_unsubscribe(Topic, SubPid, SubOpts) -> do_unsubscribe(Group, Topic, SubPid, SubOpts). do_unsubscribe(undefined, Topic, SubPid, SubOpts) -> - case maps:get(shard, SubOpts, 0) of - 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), - cast(pick(Topic), {unsubscribed, Topic}); - I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), - cast(pick({Topic, I}), {unsubscribed, Topic, I}) - end; - + clean_subscribe(SubOpts, Topic, SubPid); do_unsubscribe(Group, Topic, SubPid, _SubOpts) -> emqx_shared_sub:unsubscribe(Group, Topic, SubPid). @@ -199,7 +195,7 @@ do_unsubscribe(Group, Topic, SubPid, _SubOpts) -> -spec(publish(emqx_types:message()) -> emqx_types:publish_result()). publish(Msg) when is_record(Msg, message) -> - _ = emqx_tracer:trace(publish, Msg), + _ = emqx_tracer:trace_publish(Msg), emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'), case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of #message{headers = #{allow_publish := false}} -> @@ -231,8 +227,7 @@ delivery(Msg) -> #delivery{sender = self(), message = Msg}. -spec(route([emqx_types:route_entry()], emqx_types:delivery()) -> emqx_types:publish_result()). route([], #delivery{message = Msg}) -> - ok = emqx_hooks:run('message.dropped', [Msg, #{node => node()}, no_subscribers]), - ok = inc_dropped_cnt(Msg), + drop_message(Msg), []; route(Routes, Delivery) -> @@ -240,6 +235,10 @@ route(Routes, Delivery) -> [do_route(Route, Delivery) | Acc] end, [], Routes). +drop_message(Msg) -> + ok = emqx_hooks:run('message.dropped', [Msg, #{node => node()}, no_subscribers]), + ok = inc_dropped_cnt(Msg). + do_route({To, Node}, Delivery) when Node =:= node() -> {Node, To, dispatch(To, Delivery)}; do_route({To, Node}, Delivery) when is_atom(Node) -> @@ -261,7 +260,7 @@ aggre(Routes) -> end, [], Routes). %% @doc Forward message to another node. --spec(forward(node(), emqx_types:topic(), emqx_types:delivery(), RpcMode::sync|async) +-spec(forward(node(), emqx_types:topic(), emqx_types:delivery(), RpcMode::sync | async) -> emqx_types:deliver_result()). forward(Node, To, Delivery, async) -> case emqx_rpc:cast(To, Node, ?BROKER, dispatch, [To, Delivery]) of @@ -288,8 +287,7 @@ dispatch(Topic, #delivery{message = Msg}) -> end, 0, subscribers(Topic)), case DispN of 0 -> - ok = emqx_hooks:run('message.dropped', [Msg, #{node => node()}, no_subscribers]), - ok = inc_dropped_cnt(Msg), + drop_message(Msg), {error, no_subscribers}; _ -> {ok, DispN} @@ -336,17 +334,20 @@ subscriber_down(SubPid) -> SubOpts when is_map(SubOpts) -> _ = emqx_broker_helper:reclaim_seq(Topic), true = ets:delete(?SUBOPTION, {SubPid, Topic}), - case maps:get(shard, SubOpts, 0) of - 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), - ok = cast(pick(Topic), {unsubscribed, Topic}); - I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), - ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) - end; + clean_subscribe(SubOpts, Topic, SubPid); undefined -> ok end end, lookup_value(?SUBSCRIPTION, SubPid, [])), ets:delete(?SUBSCRIPTION, SubPid). +clean_subscribe(SubOpts, Topic, SubPid) -> + case maps:get(shard, SubOpts, 0) of + 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + ok = cast(pick(Topic), {unsubscribed, Topic}); + I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) + end. + %%-------------------------------------------------------------------- %% Management APIs %%-------------------------------------------------------------------- @@ -366,14 +367,14 @@ subscriptions(SubId) -> -spec(subscribed(pid() | emqx_types:subid(), emqx_topic:topic()) -> boolean()). subscribed(SubPid, Topic) when is_pid(SubPid) -> ets:member(?SUBOPTION, {SubPid, Topic}); -subscribed(SubId, Topic) when ?is_subid(SubId) -> +subscribed(SubId, Topic) when ?IS_SUBID(SubId) -> SubPid = emqx_broker_helper:lookup_subpid(SubId), ets:member(?SUBOPTION, {SubPid, Topic}). -spec(get_subopts(pid(), emqx_topic:topic()) -> maybe(emqx_types:subopts())). get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) -> lookup_value(?SUBOPTION, {SubPid, Topic}); -get_subopts(SubId, Topic) when ?is_subid(SubId) -> +get_subopts(SubId, Topic) when ?IS_SUBID(SubId) -> case emqx_broker_helper:lookup_subpid(SubId) of SubPid when is_pid(SubPid) -> get_subopts(SubPid, Topic); @@ -498,4 +499,3 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- - diff --git a/src/emqx_guid.erl b/src/emqx_guid.erl index 3b66f6e92..4963c48d4 100644 --- a/src/emqx_guid.erl +++ b/src/emqx_guid.erl @@ -39,6 +39,8 @@ , from_base62/1 ]). +-elvis([{elvis_style, dont_repeat_yourself, disable}]). + -define(TAG_VERSION, 131). -define(PID_EXT, 103). -define(NEW_PID_EXT, 88). @@ -137,7 +139,7 @@ npid() -> NPid. to_hexstr(I) when byte_size(I) =:= 16 -> - emqx_misc:bin2hexstr_A_F(I). + emqx_misc:bin2hexstr_a_f_upper(I). from_hexstr(S) when byte_size(S) =:= 32 -> emqx_misc:hexstr2bin(S). diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl index eb6a25377..2b77d0455 100644 --- a/src/emqx_misc.erl +++ b/src/emqx_misc.erl @@ -21,6 +21,8 @@ -include("types.hrl"). -include("logger.hrl"). +-elvis([{elvis_style, god_modules, disable}]). + -export([ merge_opts/2 , maybe_apply/2 , compose/1 @@ -47,8 +49,8 @@ , ipv6_probe/1 ]). --export([ bin2hexstr_A_F/1 - , bin2hexstr_a_f/1 +-export([ bin2hexstr_a_f_upper/1 + , bin2hexstr_a_f_lower/1 , hexstr2bin/1 ]). @@ -98,9 +100,9 @@ maybe_apply(Fun, Arg) when is_function(Fun) -> -spec(compose(list(F)) -> G when F :: fun((any()) -> any()), G :: fun((any()) -> any())). -compose([F|More]) -> compose(F, More). +compose([F | More]) -> compose(F, More). --spec(compose(F, G|[Gs]) -> C +-spec(compose(F, G | [Gs]) -> C when F :: fun((X1) -> X2), G :: fun((X2) -> X3), Gs :: [fun((Xn) -> Xn1)], @@ -108,19 +110,19 @@ compose([F|More]) -> compose(F, More). X3 :: any(), Xn :: any(), Xn1 :: any(), Xm :: any()). compose(F, G) when is_function(G) -> fun(X) -> G(F(X)) end; compose(F, [G]) -> compose(F, G); -compose(F, [G|More]) -> compose(compose(F, G), More). +compose(F, [G | More]) -> compose(compose(F, G), More). %% @doc RunFold run_fold([], Acc, _State) -> Acc; -run_fold([Fun|More], Acc, State) -> +run_fold([Fun | More], Acc, State) -> run_fold(More, Fun(Acc, State), State). %% @doc Pipeline pipeline([], Input, State) -> {ok, Input, State}; -pipeline([Fun|More], Input, State) -> +pipeline([Fun | More], Input, State) -> case apply_fun(Fun, Input, State) of ok -> pipeline(More, Input, State); {ok, NState} -> @@ -169,7 +171,7 @@ drain_deliver(0, Acc) -> drain_deliver(N, Acc) -> receive Deliver = {deliver, _Topic, _Msg} -> - drain_deliver(N-1, [Deliver|Acc]) + drain_deliver(N-1, [Deliver | Acc]) after 0 -> lists:reverse(Acc) end. @@ -184,7 +186,7 @@ drain_down(0, Acc) -> drain_down(Cnt, Acc) -> receive {'DOWN', _MRef, process, Pid, _Reason} -> - drain_down(Cnt-1, [Pid|Acc]) + drain_down(Cnt-1, [Pid | Acc]) after 0 -> lists:reverse(Acc) end. @@ -210,7 +212,7 @@ check_oom(Pid, #{message_queue_len := MaxQLen, end. do_check_oom([]) -> ok; -do_check_oom([{Val, Max, Reason}|Rest]) -> +do_check_oom([{Val, Max, Reason} | Rest]) -> case is_integer(Max) andalso (0 < Max) andalso (Max < Val) of true -> {shutdown, Reason}; false -> do_check_oom(Rest) @@ -257,8 +259,8 @@ proc_stats(Pid) -> reductions, memory]) of undefined -> []; - [{message_queue_len, Len}|ProcStats] -> - [{mailbox_len, Len}|ProcStats] + [{message_queue_len, Len} | ProcStats] -> + [{mailbox_len, Len} | ProcStats] end. rand_seed() -> @@ -278,17 +280,17 @@ index_of(E, L) -> index_of(_E, _I, []) -> error(badarg); -index_of(E, I, [E|_]) -> +index_of(E, I, [E | _]) -> I; -index_of(E, I, [_|L]) -> +index_of(E, I, [_ | L]) -> index_of(E, I+1, L). --spec(bin2hexstr_A_F(binary()) -> binary()). -bin2hexstr_A_F(B) when is_binary(B) -> +-spec(bin2hexstr_a_f_upper(binary()) -> binary()). +bin2hexstr_a_f_upper(B) when is_binary(B) -> << <<(int2hexchar(H, upper)), (int2hexchar(L, upper))>> || <> <= B>>. --spec(bin2hexstr_a_f(binary()) -> binary()). -bin2hexstr_a_f(B) when is_binary(B) -> +-spec(bin2hexstr_a_f_lower(binary()) -> binary()). +bin2hexstr_a_f_lower(B) when is_binary(B) -> << <<(int2hexchar(H, lower)), (int2hexchar(L, lower))>> || <> <= B>>. int2hexchar(I, _) when I >= 0 andalso I < 10 -> I + $0; diff --git a/src/emqx_packet.erl b/src/emqx_packet.erl index a4d440ba1..2a1137f57 100644 --- a/src/emqx_packet.erl +++ b/src/emqx_packet.erl @@ -46,24 +46,6 @@ -export([format/1]). --define(TYPE_NAMES, - { 'CONNECT' - , 'CONNACK' - , 'PUBLISH' - , 'PUBACK' - , 'PUBREC' - , 'PUBREL' - , 'PUBCOMP' - , 'SUBSCRIBE' - , 'SUBACK' - , 'UNSUBSCRIBE' - , 'UNSUBACK' - , 'PINGREQ' - , 'PINGRESP' - , 'DISCONNECT' - , 'AUTH' - }). - -type(connect() :: #mqtt_packet_connect{}). -type(publish() :: #mqtt_packet_publish{}). -type(subscribe() :: #mqtt_packet_subscribe{}). @@ -107,14 +89,14 @@ retain(#mqtt_packet{header = #mqtt_packet_header{retain = Retain}}) -> %%-------------------------------------------------------------------- %% @doc Protocol name of the CONNECT Packet. --spec(proto_name(emqx_types:packet()|connect()) -> binary()). +-spec(proto_name(emqx_types:packet() | connect()) -> binary()). proto_name(?CONNECT_PACKET(ConnPkt)) -> proto_name(ConnPkt); proto_name(#mqtt_packet_connect{proto_name = Name}) -> Name. %% @doc Protocol version of the CONNECT Packet. --spec(proto_ver(emqx_types:packet()|connect()) -> emqx_types:version()). +-spec(proto_ver(emqx_types:packet() | connect()) -> emqx_types:version()). proto_ver(?CONNECT_PACKET(ConnPkt)) -> proto_ver(ConnPkt); proto_ver(#mqtt_packet_connect{proto_ver = Ver}) -> @@ -249,7 +231,7 @@ set_props(Props, #mqtt_packet_auth{} = Pkt) -> %%-------------------------------------------------------------------- %% @doc Check PubSub Packet. --spec(check(emqx_types:packet()|publish()|subscribe()|unsubscribe()) +-spec(check(emqx_types:packet() | publish() | subscribe() | unsubscribe()) -> ok | {error, emqx_types:reason_code()}). check(#mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH}, variable = PubPkt}) when not is_tuple(PubPkt) -> @@ -318,7 +300,7 @@ check_pub_props(#{'Response-Topic' := ResponseTopic}) -> check_pub_props(_Props) -> ok. %% @doc Check CONNECT Packet. --spec(check(emqx_types:packet()|connect(), Opts :: map()) +-spec(check(emqx_types:packet() | connect(), Opts :: map()) -> ok | {error, emqx_types:reason_code()}). check(?CONNECT_PACKET(ConnPkt), Opts) -> check(ConnPkt, Opts); @@ -357,11 +339,13 @@ check_conn_props(#mqtt_packet_connect{properties = undefined}, _Opts) -> ok; check_conn_props(#mqtt_packet_connect{properties = #{'Receive-Maximum' := 0}}, _Opts) -> {error, ?RC_PROTOCOL_ERROR}; -check_conn_props(#mqtt_packet_connect{properties = #{'Request-Response-Information' := ReqRespInfo}}, _Opts) - when ReqRespInfo =/= 0, ReqRespInfo =/= 1 -> +check_conn_props(#mqtt_packet_connect{properties = + #{'Request-Response-Information' := ReqRespInfo}}, _Opts) + when ReqRespInfo =/= 0, ReqRespInfo =/= 1 -> {error, ?RC_PROTOCOL_ERROR}; -check_conn_props(#mqtt_packet_connect{properties = #{'Request-Problem-Information' := ReqProInfo}}, _Opts) - when ReqProInfo =/= 0, ReqProInfo =/= 1 -> +check_conn_props(#mqtt_packet_connect{properties = + #{'Request-Problem-Information' := ReqProInfo}}, _Opts) + when ReqProInfo =/= 0, ReqProInfo =/= 1 -> {error, ?RC_PROTOCOL_ERROR}; check_conn_props(_ConnPkt, _Opts) -> ok. @@ -382,7 +366,7 @@ check_will_msg(#mqtt_packet_connect{will_topic = WillTopic}, _Opts) -> run_checks([], _Packet, _Options) -> ok; -run_checks([Check|More], Packet, Options) -> +run_checks([Check | More], Packet, Options) -> case Check(Packet, Options) of ok -> run_checks(More, Packet, Options); Error = {error, _Reason} -> Error @@ -419,7 +403,8 @@ to_message(#mqtt_packet{ Msg#message{flags = #{dup => Dup, retain => Retain}, headers = Headers#{properties => Props}}. --spec(will_msg(#mqtt_packet_connect{}) -> emqx_types:message()). +-type(connectPacket() :: #mqtt_packet_connect{}). +-spec(will_msg(connectPacket()) -> emqx_types:message()). will_msg(#mqtt_packet_connect{will_flag = false}) -> undefined; will_msg(#mqtt_packet_connect{clientid = ClientId, @@ -468,13 +453,16 @@ format_variable(#mqtt_packet_connect{ will_payload = WillPayload, username = Username, password = Password}) -> - Format = "ClientId=~s, ProtoName=~s, ProtoVsn=~p, CleanStart=~s, KeepAlive=~p, Username=~s, Password=~s", - Args = [ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)], - {Format1, Args1} = if - WillFlag -> {Format ++ ", Will(Q~p, R~p, Topic=~s, Payload=~0p)", - Args ++ [WillQoS, i(WillRetain), WillTopic, WillPayload]}; - true -> {Format, Args} - end, + Format = "ClientId=~s, ProtoName=~s, ProtoVsn=~p, CleanStart=~s," + " KeepAlive=~p, Username=~s, Password=~s", + Args = [ClientId, ProtoName, ProtoVer, CleanStart, + KeepAlive, Username, format_password(Password)], + {Format1, Args1} = + case WillFlag of + true -> {Format ++ ", Will(Q~p, R~p, Topic=~s, Payload=~0p)", + Args ++ [WillQoS, i(WillRetain), WillTopic, WillPayload]}; + false -> {Format, Args} + end, io_lib:format(Format1, Args1); format_variable(#mqtt_packet_disconnect @@ -520,4 +508,3 @@ format_password(_Password) -> '******'. i(true) -> 1; i(false) -> 0; i(I) when is_integer(I) -> I. - diff --git a/src/emqx_tracer.erl b/src/emqx_tracer.erl index 995712f6c..e563c8eab 100644 --- a/src/emqx_tracer.erl +++ b/src/emqx_tracer.erl @@ -22,13 +22,29 @@ -logger_header("[Tracer]"). %% APIs --export([ trace/2 +-export([ trace_publish/1 + , trace_subscribe/3 + , trace_unsubscribe/2 , start_trace/3 + , start_trace/4 , lookup_traces/0 - , stop_trace/1 + , stop_trace/3 + , stop_trace/2 ]). --type(trace_who() :: {clientid | topic, binary()}). +-ifdef(TEST). +-export([is_match/3]). +-endif. + +-type(label() :: 'CONNECT' | 'CONNACK' | 'PUBLISH' | 'PUBACK' | 'PUBREC' | + 'PUBREL' | 'PUBCOMP' | 'SUBSCRIBE' | 'SUBACK' | 'UNSUBSCRIBE' | + 'UNSUBACK' | 'PINGREQ' | 'PINGRESP' | 'DISCONNECT' | 'AUTH'). + +-type(tracer() :: #{name := binary(), + type := clientid | topic, + clientid => emqx_types:clientid(), + topic => emqx_types:topic(), + labels := [label()]}). -define(TRACER, ?MODULE). -define(FORMAT, {logger_formatter, @@ -44,10 +60,16 @@ msg, "\n"], single_line => false }}). --define(TOPIC_TRACE_ID(T), "trace_topic_"++T). --define(CLIENT_TRACE_ID(C), "trace_clientid_"++C). --define(TOPIC_TRACE(T), {topic, T}). --define(CLIENT_TRACE(C), {clientid, C}). +-define(TOPIC_COMBINATOR, <<"_trace_topic_">>). +-define(CLIENTID_COMBINATOR, <<"_trace_clientid_">>). +-define(TOPIC_TRACE_ID(T, N), + binary_to_atom(<<(N)/binary, ?TOPIC_COMBINATOR/binary, (T)/binary>>)). +-define(CLIENT_TRACE_ID(C, N), + binary_to_atom(<<(N)/binary, ?CLIENTID_COMBINATOR/binary, (C)/binary>>)). +-define(TOPIC_TRACE(T, N, M), {topic, T, N, M}). +-define(CLIENT_TRACE(C, N, M), {clientid, C, N, M}). +-define(TOPIC_TRACE(T, N), {topic, T, N}). +-define(CLIENT_TRACE(C, N), {clientid, C, N}). -define(IS_LOG_LEVEL(L), L =:= emergency orelse @@ -59,28 +81,49 @@ L =:= info orelse L =:= debug). --dialyzer({nowarn_function, [install_trace_handler/3]}). - %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ -trace(publish, #message{topic = <<"$SYS/", _/binary>>}) -> +trace_publish(#message{topic = <<"$SYS/", _/binary>>}) -> %% Do not trace '$SYS' publish ignore; -trace(publish, #message{from = From, topic = Topic, payload = Payload}) +trace_publish(#message{from = From, topic = Topic, payload = Payload}) when is_binary(From); is_atom(From) -> emqx_logger:info(#{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} }, "PUBLISH to ~s: ~0p", [Topic, Payload]). +trace_subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore; +trace_subscribe(Topic, SubId, SubOpts) -> + emqx_logger:info(#{topic => Topic, + mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, + "~ts SUBSCRIBE ~ts: Options: ~0p", [SubId, Topic, SubOpts]). + +trace_unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore; +trace_unsubscribe(Topic, SubOpts) -> + emqx_logger:info(#{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, + "~ts UNSUBSCRIBE ~ts: Options: ~0p", + [maps:get(subid, SubOpts, ""), Topic, SubOpts]). + +-spec(start_trace(clientid | topic, emqx_types:clientid() | emqx_types:topic(), + logger:level() | all, string()) -> ok | {error, term()}). +start_trace(clientid, ClientId0, Level, LogFile) -> + ClientId = ensure_bin(ClientId0), + Who = #{type => clientid, clientid => ClientId, name => ClientId, labels => []}, + start_trace(Who, Level, LogFile); +start_trace(topic, Topic0, Level, LogFile) -> + Topic = ensure_bin(Topic0), + Who = #{type => topic, topic => Topic, name => Topic, labels => []}, + start_trace(Who, Level, LogFile). + %% @doc Start to trace clientid or topic. --spec(start_trace(trace_who(), logger:level() | all, string()) -> ok | {error, term()}). +-spec(start_trace(tracer(), logger:level() | all, string()) -> ok | {error, term()}). start_trace(Who, all, LogFile) -> start_trace(Who, debug, LogFile); start_trace(Who, Level, LogFile) -> case ?IS_LOG_LEVEL(Level) of true -> - #{level := PrimaryLevel} = logger:get_primary_config(), + PrimaryLevel = emqx_logger:get_primary_log_level(), try logger:compare_levels(Level, PrimaryLevel) of lt -> {error, @@ -96,13 +139,28 @@ start_trace(Who, Level, LogFile) -> false -> {error, {invalid_log_level, Level}} end. +-spec(stop_trace(clientid | topic, emqx_types:clientid() | emqx_types:topic()) -> + ok | {error, term()}). +stop_trace(Type, ClientIdOrTopic) -> + stop_trace(Type, ClientIdOrTopic, ClientIdOrTopic). + %% @doc Stop tracing clientid or topic. --spec(stop_trace(trace_who()) -> ok | {error, term()}). -stop_trace(Who) -> +-spec(stop_trace(clientid | topic, emqx_types:clientid() | emqx_types:topic(), binary()) -> + ok | {error, term()}). +stop_trace(clientid, ClientId, Name) -> + Who = #{type => clientid, clientid => ensure_bin(ClientId), name => ensure_bin(Name)}, + uninstall_trance_handler(Who); +stop_trace(topic, Topic, Name) -> + Who = #{type => topic, topic => ensure_bin(Topic), name => ensure_bin(Name)}, uninstall_trance_handler(Who). %% @doc Lookup all traces --spec(lookup_traces() -> [{Who :: trace_who(), LogFile :: string()}]). +-spec(lookup_traces() -> [#{ type => topic | clientid, + name => binary(), + topic => emqx_types:topic(), + level => logger:level(), + dst => file:filename() | console | unknown + }]). lookup_traces() -> lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)). @@ -116,7 +174,8 @@ install_trace_handler(Who, Level, LogFile) -> {fun filter_by_meta_key/2, Who}}]}) of ok -> - ?LOG(info, "Start trace for ~p", [Who]); + ?LOG(info, "Start trace for ~p", [Who]), + ok; {error, Reason} -> ?LOG(error, "Start trace for ~p failed, error: ~p", [Who, Reason]), {error, Reason} @@ -125,44 +184,78 @@ install_trace_handler(Who, Level, LogFile) -> uninstall_trance_handler(Who) -> case logger:remove_handler(handler_id(Who)) of ok -> - ?LOG(info, "Stop trace for ~p", [Who]); + ?LOG(info, "Stop trace for ~p", [Who]), + ok; {error, Reason} -> ?LOG(error, "Stop trace for ~p failed, error: ~p", [Who, Reason]), {error, Reason} end. filter_traces(#{id := Id, level := Level, dst := Dst}, Acc) -> - case atom_to_list(Id) of - ?TOPIC_TRACE_ID(T)-> - [{?TOPIC_TRACE(T), {Level, Dst}} | Acc]; - ?CLIENT_TRACE_ID(C) -> - [{?CLIENT_TRACE(C), {Level, Dst}} | Acc]; - _ -> Acc + IdStr = atom_to_binary(Id), + case binary:split(IdStr, [?TOPIC_COMBINATOR]) of + [Name, Topic] -> + [#{ type => topic, + name => Name, + topic => Topic, + level => Level, + dst => Dst} | Acc]; + _ -> + case binary:split(IdStr, [?CLIENTID_COMBINATOR]) of + [Name, ClientId] -> + [#{ type => clientid, + name => Name, + clientid => ClientId, + level => Level, + dst => Dst} | Acc]; + _ -> Acc + end end. -handler_id(?TOPIC_TRACE(Topic)) -> - list_to_atom(?TOPIC_TRACE_ID(handler_name(Topic))); -handler_id(?CLIENT_TRACE(ClientId)) -> - list_to_atom(?CLIENT_TRACE_ID(handler_name(ClientId))). +%% Plan to support topic_and_client type, so we have type field. +handler_id(#{type := topic, topic := Topic, name := Name}) -> + ?TOPIC_TRACE_ID(format(Topic), format(Name)); +handler_id(#{type := clientid, clientid := ClientId, name := Name}) -> + ?CLIENT_TRACE_ID(format(ClientId), format(Name)). -filter_by_meta_key(#{meta := Meta} = Log, {Key, Value}) -> - case is_meta_match(Key, Value, Meta) of +filter_by_meta_key(#{meta := Meta, level := Level} = Log, Context) -> + case is_match(Context, Meta, Level) of true -> Log; false -> ignore end. -is_meta_match(clientid, ClientId, #{clientid := ClientIdStr}) -> - ClientId =:= iolist_to_binary(ClientIdStr); -is_meta_match(topic, TopicFilter, #{topic := TopicMeta}) -> - emqx_topic:match(TopicMeta, TopicFilter); -is_meta_match(_, _, _) -> +%% When the log level is higher than debug and clientid/topic is match, +%% it will be logged without judging the content inside the labels. +%% When the log level is debug, in addition to the matched clientid/topic, +%% you also need to determine whether the label is in the labels +is_match(#{type := clientid, clientid := ExpectId, labels := Labels}, + #{clientid := RealId} = Meta, + Level) -> + is_match(ExpectId =:= iolist_to_binary(RealId), Level, Meta, Labels); +is_match(#{type := topic, topic := TopicFilter, labels := Labels}, + #{topic := Topic} = Meta, Level) -> + is_match(emqx_topic:match(Topic, TopicFilter), Level, Meta, Labels); +is_match(_, _, _) -> false. -handler_name(Bin) -> - case byte_size(Bin) of - Size when Size =< 200 -> binary_to_list(Bin); - _ -> hashstr(Bin) +is_match(true, debug, Meta, Labels) -> is_match_labels(Meta, Labels); +is_match(Boolean, _, _Meta, _Labels) -> Boolean. + +is_match_labels(#{trace_label := 'ALL'}, _Context) -> true; +is_match_labels(_, []) -> true; +is_match_labels(#{trace_label := Packet}, Context) -> + lists:member(Packet, Context); +is_match_labels(_, _) -> false. + +format(List)when is_list(List) -> + format(list_to_binary(List)); +format(Atom)when is_atom(Atom) -> + format(atom_to_list(Atom)); +format(Bin0)when is_binary(Bin0) -> + case byte_size(Bin0) of + Size when Size =< 200 -> Bin0; + _ -> emqx_misc:bin2hexstr_a_f_upper(Bin0) end. -hashstr(Bin) -> - binary_to_list(emqx_misc:bin2hexstr_A_F(Bin)). +ensure_bin(List) when is_list(List) -> iolist_to_binary(List); +ensure_bin(Bin) when is_binary(Bin) -> Bin. diff --git a/test/emqx_tracer_SUITE.erl b/test/emqx_tracer_SUITE.erl index f5af65440..5f7105774 100644 --- a/test/emqx_tracer_SUITE.erl +++ b/test/emqx_tracer_SUITE.erl @@ -22,8 +22,13 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-define(CLIENT, [{host, "localhost"}, + {clientid, <<"client">>}, + {username, <<"testuser">>}, + {password, <<"pass">>} + ]). -all() -> [t_trace_clientid, t_trace_topic]. +all() -> [t_trace_clientid, t_trace_topic, t_is_match]. init_per_suite(Config) -> emqx_ct_helpers:boot_modules(all), @@ -34,33 +39,36 @@ end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]). t_trace_clientid(_Config) -> - {ok, T} = emqtt:start_link([{host, "localhost"}, - {clientid, <<"client">>}, - {username, <<"testuser">>}, - {password, <<"pass">>} - ]), + {ok, T} = emqtt:start_link(?CLIENT), emqtt:connect(T), %% Start tracing emqx_logger:set_log_level(error), - {error, _} = emqx_tracer:start_trace({clientid, <<"client">>}, debug, "tmp/client.log"), + {error, _} = emqx_tracer:start_trace(clientid, <<"client">>, debug, "tmp/client.log"), emqx_logger:set_log_level(debug), - ok = emqx_tracer:start_trace({clientid, <<"client">>}, debug, "tmp/client.log"), - ok = emqx_tracer:start_trace({clientid, <<"client2">>}, all, "tmp/client2.log"), - ok = emqx_tracer:start_trace({clientid, <<"client3">>}, all, "tmp/client3.log"), - {error, {invalid_log_level, bad_level}} = emqx_tracer:start_trace({clientid, <<"client4">>}, bad_level, "tmp/client4.log"), - {error, {handler_not_added, {file_error,".",eisdir}}} = emqx_tracer:start_trace({clientid, <<"client5">>}, debug, "."), + %% add list clientid + ok = emqx_tracer:start_trace(clientid, "client", debug, "tmp/client.log"), + ok = emqx_tracer:start_trace(clientid, <<"client2">>, all, "tmp/client2.log"), + ok = emqx_tracer:start_trace(clientid, <<"client3">>, all, "tmp/client3.log"), + {error, {invalid_log_level, bad_level}} = + emqx_tracer:start_trace(clientid, <<"client4">>, bad_level, "tmp/client4.log"), + {error, {handler_not_added, {file_error, ".", eisdir}}} = + emqx_tracer:start_trace(clientid, <<"client5">>, debug, "."), ct:sleep(100), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/client.log")), ?assert(filelib:is_regular("tmp/client2.log")), + ?assert(filelib:is_regular("tmp/client3.log")), %% Get current traces - ?assertEqual([{{clientid,"client"},{debug,"tmp/client.log"}}, - {{clientid,"client2"},{debug,"tmp/client2.log"}}, - {{clientid,"client3"},{debug,"tmp/client3.log"}} - ], emqx_tracer:lookup_traces()), + ?assertEqual([#{type => clientid, clientid => <<"client">>, + name => <<"client">>, level => debug, dst => "tmp/client.log"}, + #{type => clientid, clientid => <<"client2">>, + name => <<"client2">>, level => debug, dst => "tmp/client2.log"}, + #{type => clientid, clientid => <<"client3">>, + name => <<"client3">>, level => debug, dst => "tmp/client3.log"} + ], emqx_tracer:lookup_traces()), %% set the overall log level to debug emqx_logger:set_log_level(debug), @@ -74,47 +82,90 @@ t_trace_clientid(_Config) -> ?assert(filelib:file_size("tmp/client2.log") == 0), %% Stop tracing - ok = emqx_tracer:stop_trace({clientid, <<"client">>}), - ok = emqx_tracer:stop_trace({clientid, <<"client2">>}), - ok = emqx_tracer:stop_trace({clientid, <<"client3">>}), + ok = emqx_tracer:stop_trace(clientid, <<"client">>), + ok = emqx_tracer:stop_trace(clientid, <<"client2">>), + ok = emqx_tracer:stop_trace(clientid, <<"client3">>), emqtt:disconnect(T), emqx_logger:set_log_level(warning). t_trace_topic(_Config) -> - {ok, T} = emqtt:start_link([{host, "localhost"}, - {clientid, <<"client">>}, - {username, <<"testuser">>}, - {password, <<"pass">>} - ]), + {ok, T} = emqtt:start_link(?CLIENT), emqtt:connect(T), %% Start tracing emqx_logger:set_log_level(debug), - ok = emqx_tracer:start_trace({topic, <<"x/#">>}, all, "tmp/topic_trace.log"), - ok = emqx_tracer:start_trace({topic, <<"y/#">>}, all, "tmp/topic_trace.log"), + ok = emqx_tracer:start_trace(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"), + ok = emqx_tracer:start_trace(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"), ct:sleep(100), %% Verify the tracing file exits - ?assert(filelib:is_regular("tmp/topic_trace.log")), + ?assert(filelib:is_regular("tmp/topic_trace_x.log")), + ?assert(filelib:is_regular("tmp/topic_trace_y.log")), %% Get current traces - ?assertEqual([{{topic,"x/#"},{debug,"tmp/topic_trace.log"}}, - {{topic,"y/#"},{debug,"tmp/topic_trace.log"}}], emqx_tracer:lookup_traces()), + ?assertEqual([#{type => topic, topic => <<"x/#">>, name => <<"x/#">>, + level => debug, dst => "tmp/topic_trace_x.log"}, + #{type => topic, topic => <<"y/#">>, name => <<"y/#">>, + level => debug, dst => "tmp/topic_trace_y.log"}], + emqx_tracer:lookup_traces()), %% set the overall log level to debug emqx_logger:set_log_level(debug), %% Client with clientid = "client" publishes a "hi" message to "x/y/z". - emqtt:publish(T, <<"x/y/z">>, <<"hi">>), + emqtt:publish(T, <<"x/y/z">>, <<"hi1">>), + emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), ct:sleep(200), - ?assert(filelib:file_size("tmp/topic_trace.log") > 0), + ?assert(filelib:file_size("tmp/topic_trace_x.log") > 0), + ?assert(filelib:file_size("tmp/topic_trace_y.log") =:= 0), %% Stop tracing - ok = emqx_tracer:stop_trace({topic, <<"x/#">>}), - ok = emqx_tracer:stop_trace({topic, <<"y/#">>}), - {error, _Reason} = emqx_tracer:stop_trace({topic, <<"z/#">>}), + ok = emqx_tracer:stop_trace(topic, <<"x/#">>), + ok = emqx_tracer:stop_trace(topic, <<"y/#">>), + {error, _Reason} = emqx_tracer:stop_trace(topic, <<"z/#">>), emqtt:disconnect(T), emqx_logger:set_log_level(warning). + +t_is_match(_Config) -> + ClientId = <<"test">>, + ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, + #{clientid => ClientId}, warning)), + ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, + #{clientid => ClientId}, debug)), + ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, + labels => ['PUBLISH']}, #{clientid => ClientId}, debug)), + ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, + #{clientid => ClientId, trace_label => 'PUBLISH'}, debug)), + ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => ['PUBLISH']}, + #{clientid => ClientId, trace_label => 'PUBLISH'}, debug)), + ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => ['SUBACK']}, + #{clientid => ClientId, trace_label => 'PUBLISH'}, debug)), + ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => ['SUBACK']}, + #{clientid => ClientId, trace_label => 'ALL'}, debug)), + ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, + #{clientid => <<"Bad">>}, warning)), + ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, + #{clientid => <<"Bad">>, trace_label => 'PUBLISH'}, debug)), + + Topic = <<"/test/#">>, + ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, + #{topic => <<"/test/1">>}, warning)), + ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, + #{topic => <<"/test/1/2">>}, debug)), + ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['SUBSCRIBE']}, + #{topic => <<"/test/1/2">>}, debug)), + ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, + #{topic => <<"/test/3">>, trace_label => 'PUBLISH'}, debug)), + ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['PUBLISH']}, + #{topic => <<"/test/398/">>, trace_label => 'PUBLISH'}, debug)), + ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['SUBACK']}, + #{topic => <<"/test/1/xy/y">>, trace_label => 'PUBLISH'}, debug)), + + ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['PUBLISH']}, + #{topic => <<"/t1est/398/">>, trace_label => 'PUBLISH'}, debug)), + ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, + #{topic => <<"/t1est/1/xy/y">>, trace_label => 'PUBLISH'}, debug)), + ok. From 9d4f2916c2445e1a015a2dc6ba81f576353173ef Mon Sep 17 00:00:00 2001 From: lafirest Date: Thu, 11 Nov 2021 23:49:54 +0800 Subject: [PATCH 016/104] refactor(emqx_st_statistics): optimize the code directory structure (#6128) --- .../include/emqx_st_statistics.hrl | 0 .../src/emqx_plugin_libs.app.src | 2 +- .../emqx_st_statistics/emqx_st_statistics.erl | 26 +--------- .../emqx_st_statistics_api.erl | 2 +- lib-ce/emqx_dashboard/src/emqx_dashboard.erl | 18 +++---- .../test/emqx_dashboard_SUITE.erl | 1 + .../src/emqx_mod_st_statistics.erl | 49 +++++++++++++++++++ .../test/emqx_st_statistics_SUITE.erl | 9 ---- .../test/emqx_st_statistics_api_SUITE.erl | 11 +++-- 9 files changed, 68 insertions(+), 50 deletions(-) rename {lib-ce/emqx_modules => apps/emqx_plugin_libs}/include/emqx_st_statistics.hrl (100%) rename lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl => apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl (95%) rename {lib-ce/emqx_modules => apps/emqx_plugin_libs}/src/emqx_st_statistics/emqx_st_statistics_api.erl (98%) create mode 100644 lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl diff --git a/lib-ce/emqx_modules/include/emqx_st_statistics.hrl b/apps/emqx_plugin_libs/include/emqx_st_statistics.hrl similarity index 100% rename from lib-ce/emqx_modules/include/emqx_st_statistics.hrl rename to apps/emqx_plugin_libs/include/emqx_st_statistics.hrl diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index 82937d033..f72ffc229 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.3.1"}, + {vsn, "4.3.2"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl similarity index 95% rename from lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl rename to apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl index a66975767..ac1604c9b 100644 --- a/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl +++ b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl @@ -14,9 +14,8 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mod_st_statistics). +-module(emqx_st_statistics). --behaviour(emqx_gen_mod). -behaviour(gen_server). -include_lib("include/emqx.hrl"). @@ -29,12 +28,6 @@ , disable/0, clear_history/0 ]). -%% emqx_gen_mod callbacks --export([ load/1 - , unload/1 - , description/0 - ]). - %% gen_server callbacks -export([ init/1 , handle_call/3 @@ -88,23 +81,6 @@ %% ets ordered_set is ascending by term order -%%-------------------------------------------------------------------- -%% Load/Unload -%%-------------------------------------------------------------------- - --spec(load(list()) -> ok). -load(Env) -> - emqx_mod_sup:start_child(?MODULE, worker, [Env]), - ok. - --spec(unload(list()) -> ok). -unload(_Env) -> - _ = emqx_mod_sup:stop_child(?MODULE), - ok. - -description() -> - "EMQ X Slow Topic Statistics Module". - %%-------------------------------------------------------------------- %%-------------------------------------------------------------------- %% APIs diff --git a/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_st_statistics_api.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl similarity index 98% rename from lib-ce/emqx_modules/src/emqx_st_statistics/emqx_st_statistics_api.erl rename to apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl index 328fddf28..ba13d70e2 100644 --- a/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_st_statistics_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl @@ -41,7 +41,7 @@ %%-------------------------------------------------------------------- clear_history(_Bindings, _Params) -> - ok = emqx_mod_st_statistics:clear_history(), + ok = emqx_st_statistics:clear_history(), return(ok). get_history(_Bindings, Params) -> diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl index 0390339d3..40a978a37 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl @@ -42,17 +42,16 @@ start_listeners() -> lists:foreach(fun(Listener) -> start_listener(Listener) end, listeners()). %% Start HTTP Listener -start_listener({Proto, Port, Options}) when Proto == http -> +start_listener({Proto, Port, Options}) -> Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, {"/api/v4/[...]", minirest, http_handlers()}], - minirest:start_http(listener_name(Proto), ranch_opts(Port, Options), Dispatch); - -start_listener({Proto, Port, Options}) when Proto == https -> - Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, - {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, - {"/api/v4/[...]", minirest, http_handlers()}], - minirest:start_https(listener_name(Proto), ranch_opts(Port, Options), Dispatch). + Server = listener_name(Proto), + RanchOpts = ranch_opts(Port, Options), + case Proto of + http -> minirest:start_http(Server, RanchOpts, Dispatch); + https -> minirest:start_https(Server, RanchOpts, Dispatch) + end. ranch_opts(Port, Options0) -> NumAcceptors = get_value(num_acceptors, Options0, 4), @@ -89,7 +88,7 @@ listener_name(Proto) -> http_handlers() -> Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), [{"/api/v4/", - minirest:handler(#{apps => Plugins ++ [emqx_modules], + minirest:handler(#{apps => Plugins ++ [emqx_modules, emqx_plugin_libs], filter => fun ?MODULE:filter/1}), [{authorization, fun ?MODULE:is_authorized/1}]}]. @@ -116,6 +115,7 @@ is_authorized(_Path, Req) -> _ -> false end. +filter(#{app := emqx_plugin_libs}) -> true; filter(#{app := emqx_modules}) -> true; filter(#{app := App}) -> case emqx_plugins:find_plugin(App) of diff --git a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl index ef2e747fa..550f1e9de 100644 --- a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -54,6 +54,7 @@ groups() -> ]. init_per_suite(Config) -> + application:load(emqx_plugin_libs), emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_dashboard]), Config. diff --git a/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl b/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl new file mode 100644 index 000000000..d6796122c --- /dev/null +++ b/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl @@ -0,0 +1,49 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mod_st_statistics). + +-behaviour(emqx_gen_mod). + +-include_lib("include/emqx.hrl"). +-include_lib("include/logger.hrl"). + +-logger_header("[SLOW TOPICS]"). + +%% emqx_gen_mod callbacks +-export([ load/1 + , unload/1 + , description/0 + ]). + +-define(LIB, emqx_st_statistics). + +%%-------------------------------------------------------------------- +%% Load/Unload +%%-------------------------------------------------------------------- + +-spec(load(list()) -> ok). +load(Env) -> + emqx_mod_sup:start_child(?LIB, worker, [Env]), + ok. + +-spec(unload(list()) -> ok). +unload(_Env) -> + _ = emqx_mod_sup:stop_child(?LIB), + ok. + +description() -> + "EMQ X Slow Topic Statistics Module". diff --git a/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl b/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl index 36da813f2..8df07f8b8 100644 --- a/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl @@ -65,15 +65,6 @@ t_log_and_pub(_) -> end, lists:seq(1, 10)), - timer:sleep(100), - - case ets:info(?LOG_TAB, size) of - 5 -> - ok; - _ -> - ?assert(ets:info(?TOPK_TAB, size) =/= 0) - end, - timer:sleep(2400), ?assert(ets:info(?LOG_TAB, size) =:= 0), diff --git a/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl index 8255cdd5f..3cf4dafd8 100644 --- a/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl @@ -24,11 +24,11 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx_management/include/emqx_mgmt.hrl"). --include_lib("emqx_modules/include/emqx_st_statistics.hrl"). +-include_lib("emqx_plugin_libs/include/emqx_st_statistics.hrl"). -define(CONTENT_TYPE, "application/x-www-form-urlencoded"). --define(HOST, "http://127.0.0.1:8081/"). +-define(HOST, "http://127.0.0.1:18083/"). -define(API_VERSION, "v4"). @@ -38,8 +38,9 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - application:load(emqx_modules), - emqx_ct_helpers:start_apps([emqx_management]), + emqx_ct_helpers:boot_modules(all), + application:load(emqx_plugin_libs), + emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_dashboard]), Config. end_per_suite(Config) -> @@ -55,7 +56,7 @@ end_per_testcase(_, Config) -> Config. get(Key, ResponseBody) -> - maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])). + maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])). lookup_alarm(Name, [#{<<"name">> := Name} | _More]) -> true; From 7193cd4275dd4e24871d6c244c8d55dae32a0457 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 12 Nov 2021 07:33:36 +0800 Subject: [PATCH 017/104] feat(trace): move common trace module to plugin_libs (#6127) * feat(trace): move common mod to plugin_libs * fix: elvis warning --- .../src/emqx_plugin_libs.app.src | 2 +- .../src/emqx_trace/emqx_trace.erl | 487 ++++++++++++++++++ .../src/emqx_trace/emqx_trace_api.erl | 147 ++++++ .../test/emqx_trace_SUITE.erl | 353 +++++++++++++ lib-ce/emqx_dashboard/src/emqx_dashboard.erl | 5 +- .../test/emqx_dashboard_SUITE.erl | 1 - lib-ce/emqx_modules/src/emqx_mod_trace.erl | 475 +---------------- .../emqx_modules/src/emqx_mod_trace_api.erl | 136 +---- .../test/emqx_mod_trace_SUITE.erl | 478 ----------------- .../test/emqx_mod_trace_api_SUITE.erl | 179 +++++++ 10 files changed, 1193 insertions(+), 1070 deletions(-) create mode 100644 apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl create mode 100644 apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl create mode 100644 apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl delete mode 100644 lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl create mode 100644 lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index f72ffc229..7b7dcbc07 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.3.2"}, + {vsn, "4.4.0"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl new file mode 100644 index 000000000..eb2ed7276 --- /dev/null +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -0,0 +1,487 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_trace). + +-behaviour(gen_server). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-logger_header("[Trace]"). + +%% Mnesia bootstrap +-export([mnesia/1]). + +-boot_mnesia({mnesia, [boot]}). +-copy_mnesia({mnesia, [copy]}). + +-export([ start_link/0 + , list/0 + , list/1 + , get_trace_filename/1 + , create/1 + , delete/1 + , clear/0 + , update/2 + ]). + +-export([ format/1 + , zip_dir/0 + , trace_dir/0 + , trace_file/1 + , delete_files_after_send/2 + ]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). + +-define(TRACE, ?MODULE). +-define(PACKETS, tuple_to_list(?TYPE_NAMES)). +-define(MAX_SIZE, 30). + +-ifdef(TEST). +-export([log_file/2]). +-endif. + +-record(?TRACE, + { name :: binary() | undefined | '_' + , type :: clientid | topic | undefined | '_' + , topic :: emqx_types:topic() | undefined | '_' + , clientid :: emqx_types:clientid() | undefined | '_' + , packets = [] :: list() | '_' + , enable = true :: boolean() | '_' + , start_at :: integer() | undefined | binary() | '_' + , end_at :: integer() | undefined | binary() | '_' + , log_size = #{} :: map() | '_' + }). + +mnesia(boot) -> + ok = ekka_mnesia:create_table(?TRACE, [ + {type, set}, + {disc_copies, [node()]}, + {record_name, ?TRACE}, + {attributes, record_info(fields, ?TRACE)}]); +mnesia(copy) -> + ok = ekka_mnesia:copy_table(?TRACE, disc_copies). + +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec list() -> [tuple()]. +list() -> + ets:match_object(?TRACE, #?TRACE{_ = '_'}). + +-spec list(boolean()) -> [tuple()]. +list(Enable) -> + ets:match_object(?TRACE, #?TRACE{enable = Enable, _ = '_'}). + +-spec create([{Key :: binary(), Value :: binary()}]) -> + ok | {error, {duplicate_condition, iodata()} | {already_existed, iodata()} | iodata()}. +create(Trace) -> + case mnesia:table_info(?TRACE, size) < ?MAX_SIZE of + true -> + case to_trace(Trace) of + {ok, TraceRec} -> create_new_trace(TraceRec); + {error, Reason} -> {error, Reason} + end; + false -> + {error, """The number of traces created has reached the maximum, +please delete the useless ones first"""} + end. + +-spec delete(Name :: binary()) -> ok | {error, not_found}. +delete(Name) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name) of + [_] -> mnesia:delete(?TRACE, Name, write); + [] -> mnesia:abort(not_found) + end + end, + transaction(Tran). + +-spec clear() -> ok | {error, Reason :: term()}. +clear() -> + case mnesia:clear_table(?TRACE) of + {atomic, ok} -> ok; + {aborted, Reason} -> {error, Reason} + end. + +-spec update(Name :: binary(), Enable :: boolean()) -> + ok | {error, not_found | finished}. +update(Name, Enable) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name) of + [] -> mnesia:abort(not_found); + [#?TRACE{enable = Enable}] -> ok; + [Rec] -> + case erlang:system_time(second) >= Rec#?TRACE.end_at of + false -> mnesia:write(?TRACE, Rec#?TRACE{enable = Enable}, write); + true -> mnesia:abort(finished) + end + end + end, + transaction(Tran). + +-spec get_trace_filename(Name :: binary()) -> + {ok, FileName :: string()} | {error, not_found}. +get_trace_filename(Name) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name, read) of + [] -> mnesia:abort(not_found); + [#?TRACE{start_at = Start}] -> {ok, filename(Name, Start)} + end end, + transaction(Tran). + +-spec trace_file(File :: list()) -> + {ok, Node :: list(), Binary :: binary()} | + {error, Node :: list(), Reason :: term()}. +trace_file(File) -> + FileName = filename:join(trace_dir(), File), + Node = atom_to_list(node()), + case file:read_file(FileName) of + {ok, Bin} -> {ok, Node, Bin}; + {error, Reason} -> {error, Node, Reason} + end. + +delete_files_after_send(TraceLog, Zips) -> + gen_server:cast(?MODULE, {delete_tag, self(), [TraceLog | Zips]}). + +-spec format(list(#?TRACE{})) -> list(map()). +format(Traces) -> + Fields = record_info(fields, ?TRACE), + lists:map(fun(Trace0 = #?TRACE{start_at = StartAt, end_at = EndAt}) -> + Trace = Trace0#?TRACE{ + start_at = list_to_binary(calendar:system_time_to_rfc3339(StartAt)), + end_at = list_to_binary(calendar:system_time_to_rfc3339(EndAt)) + }, + [_ | Values] = tuple_to_list(Trace), + maps:from_list(lists:zip(Fields, Values)) + end, Traces). + +init([]) -> + erlang:process_flag(trap_exit, true), + OriginLogLevel = emqx_logger:get_primary_log_level(), + ok = filelib:ensure_dir(trace_dir()), + ok = filelib:ensure_dir(zip_dir()), + {ok, _} = mnesia:subscribe({table, ?TRACE, simple}), + Traces = get_enable_trace(), + ok = update_log_primary_level(Traces, OriginLogLevel), + TRef = update_trace(Traces), + {ok, #{timer => TRef, monitors => #{}, primary_log_level => OriginLogLevel}}. + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ok, State}. + +handle_cast({delete_tag, Pid, Files}, State = #{monitors := Monitors}) -> + erlang:monitor(process, Pid), + {noreply, State#{monitors => Monitors#{Pid => Files}}}; +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitors}) -> + case maps:take(Pid, Monitors) of + error -> {noreply, State}; + {Files, NewMonitors} -> + lists:foreach(fun(F) -> file:delete(F) end, Files), + {noreply, State#{monitors => NewMonitors}} + end; +handle_info({timeout, TRef, update_trace}, + #{timer := TRef, primary_log_level := OriginLogLevel} = State) -> + Traces = get_enable_trace(), + ok = update_log_primary_level(Traces, OriginLogLevel), + NextTRef = update_trace(Traces), + {noreply, State#{timer => NextTRef}}; + +handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) -> + emqx_misc:cancel_timer(TRef), + handle_info({timeout, TRef, update_trace}, State); + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, #{timer := TRef, primary_log_level := OriginLogLevel}) -> + ok = set_log_primary_level(OriginLogLevel), + _ = mnesia:unsubscribe({table, ?TRACE, simple}), + emqx_misc:cancel_timer(TRef), + stop_all_trace_handler(), + _ = file:del_dir_r(zip_dir()), + ok. + +code_change(_, State, _Extra) -> + {ok, State}. + +create_new_trace(Trace) -> + Tran = fun() -> + case mnesia:read(?TRACE, Trace#?TRACE.name) of + [] -> + #?TRACE{start_at = StartAt, topic = Topic, + clientid = ClientId, packets = Packets} = Trace, + Match = #?TRACE{_ = '_', start_at = StartAt, topic = Topic, + clientid = ClientId, packets = Packets}, + case mnesia:match_object(?TRACE, Match, read) of + [] -> mnesia:write(?TRACE, Trace, write); + [#?TRACE{name = Name}] -> mnesia:abort({duplicate_condition, Name}) + end; + [#?TRACE{name = Name}] -> mnesia:abort({already_existed, Name}) + end + end, + transaction(Tran). + +update_trace(Traces) -> + Now = erlang:system_time(second), + {_Waiting, Running, Finished} = classify_by_time(Traces, Now), + disable_finished(Finished), + Started = already_running(), + {NeedRunning, AllStarted} = start_trace(Running, Started), + NeedStop = AllStarted -- NeedRunning, + ok = stop_trace(NeedStop, Started), + clean_stale_trace_files(), + NextTime = find_closest_time(Traces, Now), + emqx_misc:start_timer(NextTime, update_trace). + +stop_all_trace_handler() -> + lists:foreach(fun(#{type := Type, name := Name} = Trace) -> + _ = emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name) + end + , already_running()). + +already_running() -> + emqx_tracer:lookup_traces(). + +get_enable_trace() -> + {atomic, Traces} = + mnesia:transaction(fun() -> + mnesia:match_object(?TRACE, #?TRACE{enable = true, _ = '_'}, read) + end), + Traces. + +find_closest_time(Traces, Now) -> + Sec = + lists:foldl( + fun(#?TRACE{start_at = Start, end_at = End}, Closest) + when Start >= Now andalso Now < End -> %% running + min(End - Now, Closest); + (#?TRACE{start_at = Start}, Closest) when Start < Now -> %% waiting + min(Now - Start, Closest); + (_, Closest) -> Closest %% finished + end, 60 * 15, Traces), + timer:seconds(Sec). + +disable_finished([]) -> ok; +disable_finished(Traces) -> + NameWithLogSize = + lists:map(fun(#?TRACE{name = Name, start_at = StartAt}) -> + FileSize = filelib:file_size(log_file(Name, StartAt)), + {Name, FileSize} end, Traces), + transaction(fun() -> + lists:map(fun({Name, LogSize}) -> + case mnesia:read(?TRACE, Name, write) of + [] -> ok; + [Trace = #?TRACE{log_size = Logs}] -> + mnesia:write(?TRACE, Trace#?TRACE{enable = false, + log_size = Logs#{node() => LogSize}}, write) + end end, NameWithLogSize) + end). + +start_trace(Traces, Started0) -> + Started = lists:map(fun(#{name := Name}) -> Name end, Started0), + lists:foldl(fun(#?TRACE{name = Name} = Trace, {Running, StartedAcc}) -> + case lists:member(Name, StartedAcc) of + true -> {[Name | Running], StartedAcc}; + false -> + case start_trace(Trace) of + ok -> {[Name | Running], [Name | StartedAcc]}; + Error -> + ?LOG(error, "(~p)start trace failed by:~p", [Name, Error]), + {[Name | Running], StartedAcc} + end + end + end, {[], Started}, Traces). + +start_trace(Trace) -> + #?TRACE{name = Name + , type = Type + , clientid = ClientId + , topic = Topic + , packets = Packets + , start_at = Start + } = Trace, + Who0 = #{name => Name, labels => Packets}, + Who = + case Type of + topic -> Who0#{type => topic, topic => Topic}; + clientid -> Who0#{type => clientid, clientid => ClientId} + end, + case emqx_tracer:start_trace(Who, debug, log_file(Name, Start)) of + ok -> ok; + {error, {already_exist, _}} -> ok; + {error, Reason} -> {error, Reason} + end. + +stop_trace(Finished, Started) -> + lists:foreach(fun(#{name := Name, type := Type} = Trace) -> + case lists:member(Name, Finished) of + true -> emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name); + false -> ok + end + end, Started). + +clean_stale_trace_files() -> + TraceDir = trace_dir(), + case file:list_dir(TraceDir) of + {ok, AllFiles} when AllFiles =/= ["zip"] -> + FileFun = fun(#?TRACE{name = Name, start_at = StartAt}) -> filename(Name, StartAt) end, + KeepFiles = lists:map(FileFun, list()), + case AllFiles -- ["zip" | KeepFiles] of + [] -> ok; + DeleteFiles -> + DelFun = fun(F) -> file:delete(filename:join(TraceDir, F)) end, + lists:foreach(DelFun, DeleteFiles) + end; + _ -> ok + end. + +classify_by_time(Traces, Now) -> + classify_by_time(Traces, Now, [], [], []). + +classify_by_time([], _Now, Wait, Run, Finish) -> {Wait, Run, Finish}; +classify_by_time([Trace = #?TRACE{start_at = Start} | Traces], + Now, Wait, Run, Finish) when Start > Now -> + classify_by_time(Traces, Now, [Trace | Wait], Run, Finish); +classify_by_time([Trace = #?TRACE{end_at = End} | Traces], + Now, Wait, Run, Finish) when End =< Now -> + classify_by_time(Traces, Now, Wait, Run, [Trace | Finish]); +classify_by_time([Trace | Traces], Now, Wait, Run, Finish) -> + classify_by_time(Traces, Now, Wait, [Trace | Run], Finish). + +to_trace(TraceList) -> + case to_trace(TraceList, #?TRACE{}) of + {error, Reason} -> {error, Reason}; + {ok, #?TRACE{name = undefined}} -> + {error, "name required"}; + {ok, #?TRACE{type = undefined}} -> + {error, "type required"}; + {ok, #?TRACE{topic = undefined, clientid = undefined}} -> + {error, "topic/clientid cannot be both empty"}; + {ok, Trace} -> + case fill_default(Trace) of + #?TRACE{start_at = Start, end_at = End} when End =< Start -> + {error, "failed by start_at >= end_at"}; + Trace1 -> {ok, Trace1} + end + end. + +fill_default(Trace = #?TRACE{start_at = undefined}) -> + fill_default(Trace#?TRACE{start_at = erlang:system_time(second)}); +fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) -> + fill_default(Trace#?TRACE{end_at = StartAt + 10 * 60}); +fill_default(Trace) -> Trace. + +to_trace([], Rec) -> {ok, Rec}; +to_trace([{<<"name">>, Name} | Trace], Rec) -> + case binary:match(Name, [<<"/">>], []) of + nomatch -> to_trace(Trace, Rec#?TRACE{name = Name}); + _ -> {error, "name cannot contain /"} + end; +to_trace([{<<"type">>, Type} | Trace], Rec) -> + case lists:member(Type, [<<"clientid">>, <<"topic">>]) of + true -> to_trace(Trace, Rec#?TRACE{type = binary_to_existing_atom(Type)}); + false -> {error, "incorrect type: only support clientid/topic"} + end; +to_trace([{<<"topic">>, Topic} | Trace], Rec) -> + case validate_topic(Topic) of + ok -> to_trace(Trace, Rec#?TRACE{topic = Topic}); + {error, Reason} -> {error, Reason} + end; +to_trace([{<<"start_at">>, StartAt} | Trace], Rec) -> + case to_system_second(StartAt) of + {ok, Sec} -> to_trace(Trace, Rec#?TRACE{start_at = Sec}); + {error, Reason} -> {error, Reason} + end; +to_trace([{<<"end_at">>, EndAt} | Trace], Rec) -> + Now = erlang:system_time(second), + case to_system_second(EndAt) of + {ok, Sec} when Sec > Now -> + to_trace(Trace, Rec#?TRACE{end_at = Sec}); + {ok, _Sec} -> + {error, "end_at time has already passed"}; + {error, Reason} -> + {error, Reason} + end; +to_trace([{<<"clientid">>, ClientId} | Trace], Rec) -> + to_trace(Trace, Rec#?TRACE{clientid = ClientId}); +to_trace([{<<"packets">>, PacketList} | Trace], Rec) -> + case to_packets(PacketList) of + {ok, Packets} -> to_trace(Trace, Rec#?TRACE{packets = Packets}); + {error, Reason} -> {error, io_lib:format("unsupport packets: ~p", [Reason])} + end; +to_trace([Unknown | _Trace], _Rec) -> {error, io_lib:format("unknown field: ~p", [Unknown])}. + +validate_topic(TopicName) -> + try emqx_topic:validate(name, TopicName) of + true -> ok + catch + error:Error -> + {error, io_lib:format("~s invalid by ~p", [TopicName, Error])} + end. + +to_system_second(At) -> + try + Sec = calendar:rfc3339_to_system_time(binary_to_list(At), [{unit, second}]), + {ok, Sec} + catch error: {badmatch, _} -> + {error, ["The rfc3339 specification not satisfied: ", At]} + end. + +to_packets(Packets) when is_list(Packets) -> + AtomTypes = lists:map(fun(Type) -> binary_to_existing_atom(Type) end, Packets), + case lists:filter(fun(T) -> not lists:member(T, ?PACKETS) end, AtomTypes) of + [] -> {ok, AtomTypes}; + InvalidE -> {error, InvalidE} + end; +to_packets(Packets) -> {error, Packets}. + +zip_dir() -> + trace_dir() ++ "zip/". + +trace_dir() -> + filename:join(emqx:get_env(data_dir), "trace") ++ "/". + +log_file(Name, Start) -> + filename:join(trace_dir(), filename(Name, Start)). + +filename(Name, Start) -> + [Time, _] = string:split(calendar:system_time_to_rfc3339(Start), "T", leading), + lists:flatten(["trace_", binary_to_list(Name), "_", Time, ".log"]). + +transaction(Tran) -> + case mnesia:transaction(Tran) of + {atomic, Res} -> Res; + {aborted, Reason} -> {error, Reason} + end. + +update_log_primary_level([], OriginLevel) -> set_log_primary_level(OriginLevel); +update_log_primary_level(_, _) -> set_log_primary_level(debug). + +set_log_primary_level(NewLevel) -> + case NewLevel =/= emqx_logger:get_primary_log_level() of + true -> emqx_logger:set_primary_log_level(NewLevel); + false -> ok + end. diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl new file mode 100644 index 000000000..1bc71654e --- /dev/null +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -0,0 +1,147 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_trace_api). +-include_lib("emqx/include/logger.hrl"). + +%% API +-export([ list_trace/2 + , create_trace/2 + , update_trace/2 + , delete_trace/2 + , clear_traces/2 + , download_zip_log/2 + , stream_log_file/2 +]). +-export([read_trace_file/3]). + +-define(TO_BIN(_B_), iolist_to_binary(_B_)). +-define(NOT_FOUND(N), {error, 'NOT_FOUND', ?TO_BIN([N, "NOT FOUND"])}). + +list_trace(_, Params) -> + List = + case Params of + [{<<"enable">>, Enable}] -> emqx_trace:list(Enable); + _ -> emqx_trace:list() + end, + {ok, emqx_trace:format(List)}. + +create_trace(_, Param) -> + case emqx_trace:create(Param) of + ok -> ok; + {error, {already_existed, Name}} -> + {error, 'ALREADY_EXISTED', ?TO_BIN([Name, "Already Exists"])}; + {error, {duplicate_condition, Name}} -> + {error, 'DUPLICATE_CONDITION', ?TO_BIN([Name, "Duplication Condition"])}; + {error, Reason} -> + {error, 'INCORRECT_PARAMS', ?TO_BIN(Reason)} + end. + +delete_trace(#{name := Name}, _Param) -> + case emqx_trace:delete(Name) of + ok -> ok; + {error, not_found} -> ?NOT_FOUND(Name) + end. + +clear_traces(_, _) -> + emqx_trace:clear(). + +update_trace(#{name := Name, operation := Operation}, _Param) -> + Enable = case Operation of disable -> false; enable -> true end, + case emqx_trace:update(Name, Enable) of + ok -> {ok, #{enable => Enable, name => Name}}; + {error, not_found} -> ?NOT_FOUND(Name) + end. + +%% if HTTP request headers include accept-encoding: gzip and file size > 300 bytes. +%% cowboy_compress_h will auto encode gzip format. +download_zip_log(#{name := Name}, _Param) -> + case emqx_trace:get_trace_filename(Name) of + {ok, TraceLog} -> + TraceFiles = collect_trace_file(TraceLog), + ZipDir = emqx_trace:zip_dir(), + Zips = group_trace_file(ZipDir, TraceLog, TraceFiles), + ZipFileName = ZipDir ++ TraceLog, + {ok, ZipFile} = zip:zip(ZipFileName, Zips), + emqx_trace:delete_files_after_send(ZipFileName, Zips), + {ok, #{}, {sendfile, 0, filelib:file_size(ZipFile), ZipFile}}; + {error, Reason} -> + {error, Reason} + end. + +group_trace_file(ZipDir, TraceLog, TraceFiles) -> + lists:foldl(fun(Res, Acc) -> + case Res of + {ok, Node, Bin} -> + ZipName = ZipDir ++ Node ++ "-" ++ TraceLog, + ok = file:write_file(ZipName, Bin), + [ZipName | Acc]; + {error, Node, Reason} -> + ?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]), + Acc + end + end, [], TraceFiles). + +collect_trace_file(TraceLog) -> + Nodes = ekka_mnesia:running_nodes(), + {Files, BadNodes} = rpc:multicall(Nodes, emqx_trace, trace_file, [TraceLog], 60000), + BadNodes =/= [] andalso ?LOG(error, "download log rpc failed on ~p", [BadNodes]), + Files. + +%% _page as position and _limit as bytes for front-end reusing components +stream_log_file(#{name := Name}, Params) -> + Node0 = proplists:get_value(<<"node">>, Params, atom_to_binary(node())), + Position0 = proplists:get_value(<<"_page">>, Params, <<"0">>), + Bytes0 = proplists:get_value(<<"_limit">>, Params, <<"500">>), + Node = binary_to_existing_atom(Node0), + Position = binary_to_integer(Position0), + Bytes = binary_to_integer(Bytes0), + case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of + {ok, Bin} -> + Meta = #{<<"page">> => Position + byte_size(Bin), <<"limit">> => Bytes}, + {ok, #{meta => Meta, items => Bin}}; + eof -> + Meta = #{<<"page">> => Position, <<"limit">> => Bytes}, + {ok, #{meta => Meta, items => <<"">>}}; + {error, Reason} -> + logger:log(error, "read_file_failed by ~p", [{Name, Reason, Position, Bytes}]), + {error, Reason} + end. + +%% this is an rpc call for stream_log_file/2 +read_trace_file(Name, Position, Limit) -> + TraceDir = emqx_trace:trace_dir(), + {ok, AllFiles} = file:list_dir(TraceDir), + TracePrefix = "trace_" ++ binary_to_list(Name) ++ "_", + Filter = fun(FileName) -> nomatch =/= string:prefix(FileName, TracePrefix) end, + case lists:filter(Filter, AllFiles) of + [TraceFile] -> + TracePath = filename:join([TraceDir, TraceFile]), + read_file(TracePath, Position, Limit); + [] -> {error, not_found} + end. + +read_file(Path, Offset, Bytes) -> + {ok, IoDevice} = file:open(Path, [read, raw, binary]), + try + _ = case Offset of + 0 -> ok; + _ -> file:position(IoDevice, {bof, Offset}) + end, + file:read(IoDevice, Bytes) + after + file:close(IoDevice) + end. diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl new file mode 100644 index 000000000..1bd28d888 --- /dev/null +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -0,0 +1,353 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_trace_SUITE). + +%% API +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-record(emqx_trace, { + name, + type, + topic, + clientid, + packets = [], + enable = true, + start_at, + end_at, + log_size = #{} + }). + +-define(PACKETS, ['CONNECT', 'CONNACK', 'PUBLISH', 'PUBACK', 'PUBREC', 'PUBREL' + , 'PUBCOMP', 'SUBSCRIBE', 'SUBACK', 'UNSUBSCRIBE', 'UNSUBACK' + , 'PINGREQ', 'PINGRESP', 'DISCONNECT', 'AUTH']). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx_plugin_libs), + emqx_ct_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([]). + +t_base_create_delete(_Config) -> + ok = emqx_trace:clear(), + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + End = to_rfc3339(Now + 30 * 60), + Name = <<"name1">>, + ClientId = <<"test-device">>, + Packets = [atom_to_binary(E) || E <- ?PACKETS], + Trace = [ + {<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, + {<<"clientid">>, ClientId}, + {<<"packets">>, Packets}, + {<<"start_at">>, Start}, + {<<"end_at">>, End} + ], + AnotherTrace = lists:keyreplace(<<"name">>, 1, Trace, {<<"name">>, <<"AnotherName">>}), + ok = emqx_trace:create(Trace), + ?assertEqual({error, {already_existed, Name}}, emqx_trace:create(Trace)), + ?assertEqual({error, {duplicate_condition, Name}}, emqx_trace:create(AnotherTrace)), + [TraceRec] = emqx_trace:list(), + Expect = #emqx_trace{ + name = Name, + type = clientid, + topic = undefined, + clientid = ClientId, + packets = ?PACKETS, + start_at = Now, + end_at = Now + 30 * 60 + }, + ?assertEqual(Expect, TraceRec), + ExpectFormat = [ + #{ + clientid => <<"test-device">>, + enable => true, + type => clientid, + packets => ?PACKETS, + name => <<"name1">>, + start_at => Start, + end_at => End, + log_size => #{}, + topic => undefined + } + ], + ?assertEqual(ExpectFormat, emqx_trace:format([TraceRec])), + ?assertEqual(ok, emqx_trace:delete(Name)), + ?assertEqual({error, not_found}, emqx_trace:delete(Name)), + ?assertEqual([], emqx_trace:list()), + ok. + +t_create_size_max(_Config) -> + emqx_trace:clear(), + lists:map(fun(Seq) -> + Name = list_to_binary("name" ++ integer_to_list(Seq)), + Trace = [{<<"name">>, Name}, {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"topic">>, list_to_binary("/x/y/" ++ integer_to_list(Seq))}], + ok = emqx_trace:create(Trace) + end, lists:seq(1, 30)), + Trace31 = [{<<"name">>, <<"name31">>}, {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/31">>}], + {error, _} = emqx_trace:create(Trace31), + ok = emqx_trace:delete(<<"name30">>), + ok = emqx_trace:create(Trace31), + ?assertEqual(30, erlang:length(emqx_trace:list())), + ok. + +t_create_failed(_Config) -> + ok = emqx_trace:clear(), + UnknownField = [{<<"unknown">>, 12}], + {error, Reason1} = emqx_trace:create(UnknownField), + ?assertEqual(<<"unknown field: {<<\"unknown\">>,12}">>, iolist_to_binary(Reason1)), + + InvalidTopic = [{<<"topic">>, "#/#//"}], + {error, Reason2} = emqx_trace:create(InvalidTopic), + ?assertEqual(<<"#/#// invalid by function_clause">>, iolist_to_binary(Reason2)), + + InvalidStart = [{<<"start_at">>, <<"2021-12-3:12">>}], + {error, Reason3} = emqx_trace:create(InvalidStart), + ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, + iolist_to_binary(Reason3)), + + InvalidEnd = [{<<"end_at">>, <<"2021-12-3:12">>}], + {error, Reason4} = emqx_trace:create(InvalidEnd), + ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, + iolist_to_binary(Reason4)), + + InvalidPackets = [{<<"packets">>, [<<"publish">>]}], + {error, Reason5} = emqx_trace:create(InvalidPackets), + ?assertEqual(<<"unsupport packets: [publish]">>, iolist_to_binary(Reason5)), + + InvalidPackets2 = [{<<"packets">>, <<"publish">>}], + {error, Reason6} = emqx_trace:create(InvalidPackets2), + ?assertEqual(<<"unsupport packets: <<\"publish\">>">>, iolist_to_binary(Reason6)), + + {error, Reason7} = emqx_trace:create([{<<"name">>, <<"test">>}, {<<"type">>, <<"clientid">>}]), + ?assertEqual(<<"topic/clientid cannot be both empty">>, iolist_to_binary(Reason7)), + + InvalidPackets4 = [{<<"name">>, <<"/test">>}, {<<"clientid">>, <<"t">>}, + {<<"type">>, <<"clientid">>}], + {error, Reason9} = emqx_trace:create(InvalidPackets4), + ?assertEqual(<<"name cannot contain /">>, iolist_to_binary(Reason9)), + + ?assertEqual({error, "type required"}, emqx_trace:create([{<<"name">>, <<"test-name">>}, + {<<"packets">>, []}, {<<"clientid">>, <<"good">>}])), + + ?assertEqual({error, "incorrect type: only support clientid/topic"}, + emqx_trace:create([{<<"name">>, <<"test-name">>}, + {<<"packets">>, []}, {<<"clientid">>, <<"good">>}, {<<"type">>, <<"typeerror">> }])), + ok. + +t_create_default(_Config) -> + ok = emqx_trace:clear(), + {error, "name required"} = emqx_trace:create([]), + ok = emqx_trace:create([{<<"name">>, <<"test-name">>}, + {<<"type">>, <<"clientid">>}, {<<"packets">>, []}, {<<"clientid">>, <<"good">>}]), + [#emqx_trace{packets = Packets}] = emqx_trace:list(), + ?assertEqual([], Packets), + ok = emqx_trace:clear(), + Trace = [ + {<<"name">>, <<"test-name">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/z">>}, + {<<"start_at">>, <<"2021-10-28T10:54:47+08:00">>}, + {<<"end_at">>, <<"2021-10-27T10:54:47+08:00">>} + ], + {error, "end_at time has already passed"} = emqx_trace:create(Trace), + Now = erlang:system_time(second), + Trace2 = [ + {<<"name">>, <<"test-name">>}, + {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"topic">>, <<"/x/y/z">>}, + {<<"start_at">>, to_rfc3339(Now + 10)}, + {<<"end_at">>, to_rfc3339(Now + 3)} + ], + {error, "failed by start_at >= end_at"} = emqx_trace:create(Trace2), + ok = emqx_trace:create([{<<"name">>, <<"test-name">>}, + {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/z">>}]), + [#emqx_trace{start_at = Start, end_at = End}] = emqx_trace:list(), + ?assertEqual(10 * 60, End - Start), + ?assertEqual(true, Start - erlang:system_time(second) < 5), + ok. + +t_update_enable(_Config) -> + ok = emqx_trace:clear(), + Name = <<"test-name">>, + Now = erlang:system_time(second), + End = list_to_binary(calendar:system_time_to_rfc3339(Now + 2)), + ok = emqx_trace:create([{<<"name">>, Name}, {<<"packets">>, [<<"PUBLISH">>]}, + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/z">>}, {<<"end_at">>, End}]), + [#emqx_trace{enable = Enable}] = emqx_trace:list(), + ?assertEqual(Enable, true), + ok = emqx_trace:update(Name, false), + [#emqx_trace{enable = false}] = emqx_trace:list(), + ok = emqx_trace:update(Name, false), + [#emqx_trace{enable = false}] = emqx_trace:list(), + ok = emqx_trace:update(Name, true), + [#emqx_trace{enable = true}] = emqx_trace:list(), + ok = emqx_trace:update(Name, false), + [#emqx_trace{enable = false}] = emqx_trace:list(), + ?assertEqual({error, not_found}, emqx_trace:update(<<"Name not found">>, true)), + ct:sleep(2100), + ?assertEqual({error, finished}, emqx_trace:update(Name, true)), + ok. + +t_load_state(_Config) -> + emqx_trace:clear(), + load(), + Now = erlang:system_time(second), + Running = [{<<"name">>, <<"Running">>}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/1">>}, {<<"start_at">>, to_rfc3339(Now - 1)}, + {<<"end_at">>, to_rfc3339(Now + 2)}], + Waiting = [{<<"name">>, <<"Waiting">>}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/2">>}, {<<"start_at">>, to_rfc3339(Now + 3)}, + {<<"end_at">>, to_rfc3339(Now + 8)}], + Finished = [{<<"name">>, <<"Finished">>}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/3">>}, {<<"start_at">>, to_rfc3339(Now - 5)}, + {<<"end_at">>, to_rfc3339(Now)}], + ok = emqx_trace:create(Running), + ok = emqx_trace:create(Waiting), + {error, "end_at time has already passed"} = emqx_trace:create(Finished), + Traces = emqx_trace:format(emqx_trace:list()), + ?assertEqual(2, erlang:length(Traces)), + Enables = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces), + ExpectEnables = [{<<"Running">>, true}, {<<"Waiting">>, true}], + ?assertEqual(ExpectEnables, lists:sort(Enables)), + ct:sleep(3500), + Traces2 = emqx_trace:format(emqx_trace:list()), + ?assertEqual(2, erlang:length(Traces2)), + Enables2 = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces2), + ExpectEnables2 = [{<<"Running">>, false}, {<<"Waiting">>, true}], + ?assertEqual(ExpectEnables2, lists:sort(Enables2)), + unload(), + ok. + +t_client_event(_Config) -> + application:set_env(emqx, allow_anonymous, true), + emqx_trace:clear(), + ClientId = <<"client-test">>, + load(), + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + Name = <<"test_client_id_event">>, + ok = emqx_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + ct:sleep(200), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + emqtt:ping(Client), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"1">>, [{qos, 0}]), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"2">>, [{qos, 0}]), + ct:sleep(200), + ok = emqx_trace:create([{<<"name">>, <<"test_topic">>}, + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/test">>}, {<<"start_at">>, Start}]), + ct:sleep(200), + {ok, Bin} = file:read_file(emqx_trace:log_file(Name, Now)), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"3">>, [{qos, 0}]), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"4">>, [{qos, 0}]), + ok = emqtt:disconnect(Client), + ct:sleep(200), + {ok, Bin2} = file:read_file(emqx_trace:log_file(Name, Now)), + {ok, Bin3} = file:read_file(emqx_trace:log_file(<<"test_topic">>, Now)), + ct:pal("Bin ~p Bin2 ~p Bin3 ~p", [byte_size(Bin), byte_size(Bin2), byte_size(Bin3)]), + ?assert(erlang:byte_size(Bin) > 0), + ?assert(erlang:byte_size(Bin) < erlang:byte_size(Bin2)), + ?assert(erlang:byte_size(Bin3) > 0), + unload(), + ok. + +t_get_log_filename(_Config) -> + ok = emqx_trace:clear(), + load(), + Now = erlang:system_time(second), + Start = calendar:system_time_to_rfc3339(Now), + End = calendar:system_time_to_rfc3339(Now + 2), + Name = <<"name1">>, + ClientId = <<"test-device">>, + Packets = [atom_to_binary(E) || E <- ?PACKETS], + Trace = [ + {<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, + {<<"clientid">>, ClientId}, + {<<"packets">>, Packets}, + {<<"start_at">>, list_to_binary(Start)}, + {<<"end_at">>, list_to_binary(End)} + ], + ok = emqx_trace:create(Trace), + ?assertEqual({error, not_found}, emqx_trace:get_trace_filename(<<"test">>)), + ?assertEqual(ok, element(1, emqx_trace:get_trace_filename(Name))), + ct:sleep(3000), + ?assertEqual(ok, element(1, emqx_trace:get_trace_filename(Name))), + unload(), + ok. + +t_trace_file(_Config) -> + FileName = "test.log", + Content = <<"test \n test">>, + TraceDir = emqx_trace:trace_dir(), + File = filename:join(TraceDir, FileName), + ok = file:write_file(File, Content), + {ok, Node, Bin} = emqx_trace:trace_file(FileName), + ?assertEqual(Node, atom_to_list(node())), + ?assertEqual(Content, Bin), + ok = file:delete(File), + ok. + +t_download_log(_Config) -> + emqx_trace:clear(), + load(), + ClientId = <<"client-test">>, + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + Name = <<"test_client_id">>, + ok = emqx_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], + ct:sleep(100), + {ok, #{}, {sendfile, 0, ZipFileSize, _ZipFile}} = + emqx_trace_api:download_zip_log(#{name => Name}, []), + ?assert(ZipFileSize > 0), + ok = emqtt:disconnect(Client), + unload(), + ok. + +to_rfc3339(Second) -> + list_to_binary(calendar:system_time_to_rfc3339(Second)). + +load() -> + emqx_trace:start_link(). + +unload() -> + gen_server:stop(emqx_trace). diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl index 40a978a37..59318a5a1 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl @@ -41,7 +41,8 @@ start_listeners() -> lists:foreach(fun(Listener) -> start_listener(Listener) end, listeners()). -%% Start HTTP Listener + +%% Start HTTP(S) Listener start_listener({Proto, Port, Options}) -> Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, @@ -88,7 +89,7 @@ listener_name(Proto) -> http_handlers() -> Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), [{"/api/v4/", - minirest:handler(#{apps => Plugins ++ [emqx_modules, emqx_plugin_libs], + minirest:handler(#{apps => Plugins ++ [emqx_modules, emqx_plugin_libs], filter => fun ?MODULE:filter/1}), [{authorization, fun ?MODULE:is_authorized/1}]}]. diff --git a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 550f1e9de..71a7692be 100644 --- a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -166,4 +166,3 @@ api_path(Path) -> json(Data) -> {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), Jsx. - diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace.erl b/lib-ce/emqx_modules/src/emqx_mod_trace.erl index 737ee7e97..03d468a82 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_trace.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_trace.erl @@ -16,493 +16,24 @@ -module(emqx_mod_trace). --behaviour(gen_server). -behaviour(emqx_gen_mod). - -include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("kernel/include/file.hrl"). - --logger_header("[Trace]"). - -%% Mnesia bootstrap --export([mnesia/1]). - --boot_mnesia({mnesia, [boot]}). --copy_mnesia({mnesia, [copy]}). -export([ load/1 , unload/1 , description/0 ]). --export([ start_link/0 - , list/0 - , list/1 - , get_trace_filename/1 - , create/1 - , delete/1 - , clear/0 - , update/2 - ]). - --export([ format/1 - , zip_dir/0 - , trace_dir/0 - , trace_file/1 - , delete_files_after_send/2 - ]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). - --define(TRACE, ?MODULE). --define(PACKETS, tuple_to_list(?TYPE_NAMES)). --define(MAX_SIZE, 30). - --ifdef(TEST). --export([log_file/2]). --endif. - --record(?TRACE, - { name :: binary() | undefined | '_' - , type :: clientid | topic | undefined | '_' - , topic :: emqx_types:topic() | undefined | '_' - , clientid :: emqx_types:clientid() | undefined | '_' - , packets = [] :: list() | '_' - , enable = true :: boolean() | '_' - , start_at :: integer() | undefined | binary() | '_' - , end_at :: integer() | undefined | binary() | '_' - , log_size = #{} :: map() | '_' - }). - -mnesia(boot) -> - ok = ekka_mnesia:create_table(?TRACE, [ - {type, set}, - {disc_copies, [node()]}, - {record_name, ?TRACE}, - {attributes, record_info(fields, ?TRACE)}]); -mnesia(copy) -> - ok = ekka_mnesia:copy_table(?TRACE, disc_copies). - +-spec description() -> string(). description() -> "EMQ X Trace Module". -spec load(any()) -> ok. load(_Env) -> - emqx_mod_sup:start_child(?MODULE, worker). + emqx_mod_sup:start_child(emqx_trace, worker). -spec unload(any()) -> ok. unload(_Env) -> - _ = emqx_mod_sup:stop_child(?MODULE), - stop_all_trace_handler(). - --spec(start_link() -> emqx_types:startlink_ret()). -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - --spec list() -> [tuple()]. -list() -> - ets:match_object(?TRACE, #?TRACE{_ = '_'}). - --spec list(boolean()) -> [tuple()]. -list(Enable) -> - ets:match_object(?TRACE, #?TRACE{enable = Enable, _ = '_'}). - --spec create([{Key :: binary(), Value :: binary()}]) -> - ok | {error, {duplicate_condition, iodata()} | {already_existed, iodata()} | iodata()}. -create(Trace) -> - case mnesia:table_info(?TRACE, size) < ?MAX_SIZE of - true -> - case to_trace(Trace) of - {ok, TraceRec} -> create_new_trace(TraceRec); - {error, Reason} -> {error, Reason} - end; - false -> - {error, """The number of traces created has reached the maximum, -please delete the useless ones first"""} - end. - --spec delete(Name :: binary()) -> ok | {error, not_found}. -delete(Name) -> - Tran = fun() -> - case mnesia:read(?TRACE, Name) of - [_] -> mnesia:delete(?TRACE, Name, write); - [] -> mnesia:abort(not_found) - end - end, - transaction(Tran). - --spec clear() -> ok | {error, Reason :: term()}. -clear() -> - case mnesia:clear_table(?TRACE) of - {atomic, ok} -> ok; - {aborted, Reason} -> {error, Reason} - end. - --spec update(Name :: binary(), Enable :: boolean()) -> - ok | {error, not_found | finished}. -update(Name, Enable) -> - Tran = fun() -> - case mnesia:read(?TRACE, Name) of - [] -> mnesia:abort(not_found); - [#?TRACE{enable = Enable}] -> ok; - [Rec] -> - case erlang:system_time(second) >= Rec#?TRACE.end_at of - false -> mnesia:write(?TRACE, Rec#?TRACE{enable = Enable}, write); - true -> mnesia:abort(finished) - end - end - end, - transaction(Tran). - --spec get_trace_filename(Name :: binary()) -> - {ok, FileName :: string()} | {error, not_found}. -get_trace_filename(Name) -> - Tran = fun() -> - case mnesia:read(?TRACE, Name, read) of - [] -> mnesia:abort(not_found); - [#?TRACE{start_at = Start}] -> {ok, filename(Name, Start)} - end end, - transaction(Tran). - --spec trace_file(File :: list()) -> - {ok, Node :: list(), Binary :: binary()} | - {error, Node :: list(), Reason :: term()}. -trace_file(File) -> - FileName = filename:join(trace_dir(), File), - Node = atom_to_list(node()), - case file:read_file(FileName) of - {ok, Bin} -> {ok, Node, Bin}; - {error, Reason} -> {error, Node, Reason} - end. - -delete_files_after_send(TraceLog, Zips) -> - gen_server:cast(?MODULE, {delete_tag, self(), [TraceLog | Zips]}). - --spec format(list(#?TRACE{})) -> list(map()). -format(Traces) -> - Fields = record_info(fields, ?TRACE), - lists:map(fun(Trace0 = #?TRACE{start_at = StartAt, end_at = EndAt}) -> - Trace = Trace0#?TRACE{ - start_at = list_to_binary(calendar:system_time_to_rfc3339(StartAt)), - end_at = list_to_binary(calendar:system_time_to_rfc3339(EndAt)) - }, - [_ | Values] = tuple_to_list(Trace), - maps:from_list(lists:zip(Fields, Values)) - end, Traces). - -init([]) -> - erlang:process_flag(trap_exit, true), - OriginLogLevel = emqx_logger:get_primary_log_level(), - ok = filelib:ensure_dir(trace_dir()), - ok = filelib:ensure_dir(zip_dir()), - {ok, _} = mnesia:subscribe({table, ?TRACE, simple}), - Traces = get_enable_trace(), - ok = update_log_primary_level(Traces, OriginLogLevel), - TRef = update_trace(Traces), - {ok, #{timer => TRef, monitors => #{}, primary_log_level => OriginLogLevel}}. - -handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), - {reply, ok, State}. - -handle_cast({delete_tag, Pid, Files}, State = #{monitors := Monitors}) -> - erlang:monitor(process, Pid), - {noreply, State#{monitors => Monitors#{Pid => Files}}}; -handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), - {noreply, State}. - -handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitors}) -> - case maps:take(Pid, Monitors) of - error -> {noreply, State}; - {Files, NewMonitors} -> - lists:foreach(fun(F) -> file:delete(F) end, Files), - {noreply, State#{monitors => NewMonitors}} - end; -handle_info({timeout, TRef, update_trace}, - #{timer := TRef, primary_log_level := OriginLogLevel} = State) -> - Traces = get_enable_trace(), - ok = update_log_primary_level(Traces, OriginLogLevel), - NextTRef = update_trace(Traces), - {noreply, State#{timer => NextTRef}}; - -handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef), - handle_info({timeout, TRef, update_trace}, State); - -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, #{timer := TRef, primary_log_level := OriginLogLevel}) -> - ok = set_log_primary_level(OriginLogLevel), - _ = mnesia:unsubscribe({table, ?TRACE, simple}), - emqx_misc:cancel_timer(TRef), - stop_all_trace_handler(), - _ = file:del_dir_r(zip_dir()), - ok. - -code_change(_, State, _Extra) -> - {ok, State}. - -create_new_trace(Trace) -> - Tran = fun() -> - case mnesia:read(?TRACE, Trace#?TRACE.name) of - [] -> - #?TRACE{start_at = StartAt, topic = Topic, - clientid = ClientId, packets = Packets} = Trace, - Match = #?TRACE{_ = '_', start_at = StartAt, topic = Topic, - clientid = ClientId, packets = Packets}, - case mnesia:match_object(?TRACE, Match, read) of - [] -> mnesia:write(?TRACE, Trace, write); - [#?TRACE{name = Name}] -> mnesia:abort({duplicate_condition, Name}) - end; - [#?TRACE{name = Name}] -> mnesia:abort({already_existed, Name}) - end - end, - transaction(Tran). - -update_trace(Traces) -> - Now = erlang:system_time(second), - {_Waiting, Running, Finished} = classify_by_time(Traces, Now), - disable_finished(Finished), - Started = already_running(), - {NeedRunning, AllStarted} = start_trace(Running, Started), - NeedStop = AllStarted -- NeedRunning, - ok = stop_trace(NeedStop, Started), - clean_stale_trace_files(), - NextTime = find_closest_time(Traces, Now), - emqx_misc:start_timer(NextTime, update_trace). - -stop_all_trace_handler() -> - lists:foreach(fun(#{type := Type, name := Name} = Trace) -> - _ = emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name) - end - , already_running()). - -already_running() -> - emqx_tracer:lookup_traces(). - -get_enable_trace() -> - {atomic, Traces} = - mnesia:transaction(fun() -> - mnesia:match_object(?TRACE, #?TRACE{enable = true, _ = '_'}, read) - end), - Traces. - -find_closest_time(Traces, Now) -> - Sec = - lists:foldl( - fun(#?TRACE{start_at = Start, end_at = End}, Closest) - when Start >= Now andalso Now < End -> %% running - min(End - Now, Closest); - (#?TRACE{start_at = Start}, Closest) when Start < Now -> %% waiting - min(Now - Start, Closest); - (_, Closest) -> Closest %% finished - end, 60 * 15, Traces), - timer:seconds(Sec). - -disable_finished([]) -> ok; -disable_finished(Traces) -> - NameWithLogSize = - lists:map(fun(#?TRACE{name = Name, start_at = StartAt}) -> - FileSize = filelib:file_size(log_file(Name, StartAt)), - {Name, FileSize} end, Traces), - transaction(fun() -> - lists:map(fun({Name, LogSize}) -> - case mnesia:read(?TRACE, Name, write) of - [] -> ok; - [Trace = #?TRACE{log_size = Logs}] -> - mnesia:write(?TRACE, Trace#?TRACE{enable = false, - log_size = Logs#{node() => LogSize}}, write) - end end, NameWithLogSize) - end). - -start_trace(Traces, Started0) -> - Started = lists:map(fun(#{name := Name}) -> Name end, Started0), - lists:foldl(fun(#?TRACE{name = Name} = Trace, {Running, StartedAcc}) -> - case lists:member(Name, StartedAcc) of - true -> {[Name | Running], StartedAcc}; - false -> - case start_trace(Trace) of - ok -> {[Name | Running], [Name | StartedAcc]}; - Error -> - ?LOG(error, "(~p)start trace failed by:~p", [Name, Error]), - {[Name | Running], StartedAcc} - end - end - end, {[], Started}, Traces). - -start_trace(Trace) -> - #?TRACE{name = Name - , type = Type - , clientid = ClientId - , topic = Topic - , packets = Packets - , start_at = Start - } = Trace, - Who0 = #{name => Name, labels => Packets}, - Who = - case Type of - topic -> Who0#{type => topic, topic => Topic}; - clientid -> Who0#{type => clientid, clientid => ClientId} - end, - case emqx_tracer:start_trace(Who, debug, log_file(Name, Start)) of - ok -> ok; - {error, {already_exist, _}} -> ok; - {error, Reason} -> {error, Reason} - end. - -stop_trace(Finished, Started) -> - lists:foreach(fun(#{name := Name, type := Type} = Trace) -> - case lists:member(Name, Finished) of - true -> emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name); - false -> ok - end - end, Started). - -clean_stale_trace_files() -> - TraceDir = trace_dir(), - case file:list_dir(TraceDir) of - {ok, AllFiles} when AllFiles =/= ["zip"] -> - FileFun = fun(#?TRACE{name = Name, start_at = StartAt}) -> filename(Name, StartAt) end, - KeepFiles = lists:map(FileFun, list()), - case AllFiles -- ["zip" | KeepFiles] of - [] -> ok; - DeleteFiles -> - DelFun = fun(F) -> file:delete(filename:join(TraceDir, F)) end, - lists:foreach(DelFun, DeleteFiles) - end; - _ -> ok - end. - -classify_by_time(Traces, Now) -> - classify_by_time(Traces, Now, [], [], []). - -classify_by_time([], _Now, Wait, Run, Finish) -> {Wait, Run, Finish}; -classify_by_time([Trace = #?TRACE{start_at = Start} | Traces], - Now, Wait, Run, Finish) when Start > Now -> - classify_by_time(Traces, Now, [Trace | Wait], Run, Finish); -classify_by_time([Trace = #?TRACE{end_at = End} | Traces], - Now, Wait, Run, Finish) when End =< Now -> - classify_by_time(Traces, Now, Wait, Run, [Trace | Finish]); -classify_by_time([Trace | Traces], Now, Wait, Run, Finish) -> - classify_by_time(Traces, Now, Wait, [Trace | Run], Finish). - -to_trace(TraceList) -> - case to_trace(TraceList, #?TRACE{}) of - {error, Reason} -> {error, Reason}; - {ok, #?TRACE{name = undefined}} -> - {error, "name required"}; - {ok, #?TRACE{type = undefined}} -> - {error, "type required"}; - {ok, #?TRACE{topic = undefined, clientid = undefined}} -> - {error, "topic/clientid cannot be both empty"}; - {ok, Trace} -> - case fill_default(Trace) of - #?TRACE{start_at = Start, end_at = End} when End =< Start -> - {error, "failed by start_at >= end_at"}; - Trace1 -> {ok, Trace1} - end - end. - -fill_default(Trace = #?TRACE{start_at = undefined}) -> - fill_default(Trace#?TRACE{start_at = erlang:system_time(second)}); -fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) -> - fill_default(Trace#?TRACE{end_at = StartAt + 10 * 60}); -fill_default(Trace) -> Trace. - -to_trace([], Rec) -> {ok, Rec}; -to_trace([{<<"name">>, Name} | Trace], Rec) -> - case binary:match(Name, [<<"/">>], []) of - nomatch -> to_trace(Trace, Rec#?TRACE{name = Name}); - _ -> {error, "name cannot contain /"} - end; -to_trace([{<<"type">>, Type} | Trace], Rec) -> - case lists:member(Type, [<<"clientid">>, <<"topic">>]) of - true -> to_trace(Trace, Rec#?TRACE{type = binary_to_existing_atom(Type)}); - false -> {error, "incorrect type: only support clientid/topic"} - end; -to_trace([{<<"topic">>, Topic} | Trace], Rec) -> - case validate_topic(Topic) of - ok -> to_trace(Trace, Rec#?TRACE{topic = Topic}); - {error, Reason} -> {error, Reason} - end; -to_trace([{<<"start_at">>, StartAt} | Trace], Rec) -> - case to_system_second(StartAt) of - {ok, Sec} -> to_trace(Trace, Rec#?TRACE{start_at = Sec}); - {error, Reason} -> {error, Reason} - end; -to_trace([{<<"end_at">>, EndAt} | Trace], Rec) -> - Now = erlang:system_time(second), - case to_system_second(EndAt) of - {ok, Sec} when Sec > Now -> - to_trace(Trace, Rec#?TRACE{end_at = Sec}); - {ok, _Sec} -> - {error, "end_at time has already passed"}; - {error, Reason} -> - {error, Reason} - end; -to_trace([{<<"clientid">>, ClientId} | Trace], Rec) -> - to_trace(Trace, Rec#?TRACE{clientid = ClientId}); -to_trace([{<<"packets">>, PacketList} | Trace], Rec) -> - case to_packets(PacketList) of - {ok, Packets} -> to_trace(Trace, Rec#?TRACE{packets = Packets}); - {error, Reason} -> {error, io_lib:format("unsupport packets: ~p", [Reason])} - end; -to_trace([Unknown | _Trace], _Rec) -> {error, io_lib:format("unknown field: ~p", [Unknown])}. - -validate_topic(TopicName) -> - try emqx_topic:validate(name, TopicName) of - true -> ok - catch - error:Error -> - {error, io_lib:format("~s invalid by ~p", [TopicName, Error])} - end. - -to_system_second(At) -> - try - Sec = calendar:rfc3339_to_system_time(binary_to_list(At), [{unit, second}]), - {ok, Sec} - catch error: {badmatch, _} -> - {error, ["The rfc3339 specification not satisfied: ", At]} - end. - -to_packets(Packets) when is_list(Packets) -> - AtomTypes = lists:map(fun(Type) -> binary_to_existing_atom(Type) end, Packets), - case lists:filter(fun(T) -> not lists:member(T, ?PACKETS) end, AtomTypes) of - [] -> {ok, AtomTypes}; - InvalidE -> {error, InvalidE} - end; -to_packets(Packets) -> {error, Packets}. - -zip_dir() -> - trace_dir() ++ "zip/". - -trace_dir() -> - filename:join(emqx:get_env(data_dir), "trace") ++ "/". - -log_file(Name, Start) -> - filename:join(trace_dir(), filename(Name, Start)). - -filename(Name, Start) -> - [Time, _] = string:split(calendar:system_time_to_rfc3339(Start), "T", leading), - lists:flatten(["trace_", binary_to_list(Name), "_", Time, ".log"]). - -transaction(Tran) -> - case mnesia:transaction(Tran) of - {atomic, Res} -> Res; - {aborted, Reason} -> {error, Reason} - end. - -update_log_primary_level([], OriginLevel) -> set_log_primary_level(OriginLevel); -update_log_primary_level(_, _) -> set_log_primary_level(debug). - -set_log_primary_level(NewLevel) -> - case NewLevel =/= emqx_logger:get_primary_log_level() of - true -> emqx_logger:set_primary_log_level(NewLevel); - false -> ok - end. + emqx_mod_sup:stop_child(emqx_trace). diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl index 6d2813b96..a3abbedf7 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl @@ -15,7 +15,6 @@ %%-------------------------------------------------------------------- -module(emqx_mod_trace_api). --include_lib("emqx/include/logger.hrl"). %% API -export([ list_trace/2 @@ -26,10 +25,6 @@ , download_zip_log/2 , stream_log_file/2 ]). --export([read_trace_file/3]). - --define(TO_BIN(_B_), iolist_to_binary(_B_)). --define(RETURN_NOT_FOUND(N), return({error, 'NOT_FOUND', ?TO_BIN([N, "NOT FOUND"])})). -import(minirest, [return/1]). @@ -75,117 +70,26 @@ func => stream_log_file, descr => "download trace's log"}). -list_trace(_, Params) -> - List = - case Params of - [{<<"enable">>, Enable}] -> emqx_mod_trace:list(Enable); - _ -> emqx_mod_trace:list() - end, - return({ok, emqx_mod_trace:format(List)}). +list_trace(Path, Params) -> + return(emqx_trace_api:list_trace(Path, Params)). -create_trace(_, Param) -> - case emqx_mod_trace:create(Param) of - ok -> return(ok); - {error, {already_existed, Name}} -> - return({error, 'ALREADY_EXISTED', ?TO_BIN([Name, "Already Exists"])}); - {error, {duplicate_condition, Name}} -> - return({error, 'DUPLICATE_CONDITION', ?TO_BIN([Name, "Duplication Condition"])}); - {error, Reason} -> - return({error, 'INCORRECT_PARAMS', ?TO_BIN(Reason)}) +create_trace(Path, Params) -> + return(emqx_trace_api:create_trace(Path, Params)). + +delete_trace(Path, Params) -> + return(emqx_trace_api:delete_trace(Path, Params)). + +clear_traces(Path, Params) -> + return(emqx_trace_api:clear_traces(Path, Params)). + +update_trace(Path, Params) -> + return(emqx_trace_api:update_trace(Path, Params)). + +download_zip_log(Path, Params) -> + case emqx_trace_api:download_zip_log(Path, Params) of + {ok, _Header, _File}= Return -> Return; + {error, _Reason} = Err -> return(Err) end. -delete_trace(#{name := Name}, _Param) -> - case emqx_mod_trace:delete(Name) of - ok -> return(ok); - {error, not_found} -> ?RETURN_NOT_FOUND(Name) - end. - -clear_traces(_, _) -> - return(emqx_mod_trace:clear()). - -update_trace(#{name := Name, operation := Operation}, _Param) -> - Enable = case Operation of disable -> false; enable -> true end, - case emqx_mod_trace:update(Name, Enable) of - ok -> return({ok, #{enable => Enable, name => Name}}); - {error, not_found} -> ?RETURN_NOT_FOUND(Name) - end. - -%% if HTTP request headers include accept-encoding: gzip and file size > 300 bytes. -%% cowboy_compress_h will auto encode gzip format. -download_zip_log(#{name := Name}, _Param) -> - case emqx_mod_trace:get_trace_filename(Name) of - {ok, TraceLog} -> - TraceFiles = collect_trace_file(TraceLog), - ZipDir = emqx_mod_trace:zip_dir(), - Zips = group_trace_file(ZipDir, TraceLog, TraceFiles), - ZipFileName = ZipDir ++ TraceLog, - {ok, ZipFile} = zip:zip(ZipFileName, Zips), - emqx_mod_trace:delete_files_after_send(ZipFileName, Zips), - {ok, #{}, {sendfile, 0, filelib:file_size(ZipFile), ZipFile}}; - {error, Reason} -> - return({error, Reason}) - end. - -group_trace_file(ZipDir, TraceLog, TraceFiles) -> - lists:foldl(fun(Res, Acc) -> - case Res of - {ok, Node, Bin} -> - ZipName = ZipDir ++ Node ++ "-" ++ TraceLog, - ok = file:write_file(ZipName, Bin), - [ZipName | Acc]; - {error, Node, Reason} -> - ?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]), - Acc - end - end, [], TraceFiles). - -collect_trace_file(TraceLog) -> - Nodes = ekka_mnesia:running_nodes(), - {Files, BadNodes} = rpc:multicall(Nodes, emqx_mod_trace, trace_file, [TraceLog], 60000), - BadNodes =/= [] andalso ?LOG(error, "download log rpc failed on ~p", [BadNodes]), - Files. - -%% _page as position and _limit as bytes for front-end reusing components -stream_log_file(#{name := Name}, Params) -> - Node0 = proplists:get_value(<<"node">>, Params, atom_to_binary(node())), - Position0 = proplists:get_value(<<"_page">>, Params, <<"0">>), - Bytes0 = proplists:get_value(<<"_limit">>, Params, <<"500">>), - Node = binary_to_existing_atom(Node0), - Position = binary_to_integer(Position0), - Bytes = binary_to_integer(Bytes0), - case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of - {ok, Bin} -> - Meta = #{<<"page">> => Position + byte_size(Bin), <<"limit">> => Bytes}, - return({ok, #{meta => Meta, items => Bin}}); - eof -> - Meta = #{<<"page">> => Position, <<"limit">> => Bytes}, - return({ok, #{meta => Meta, items => <<"">>}}); - {error, Reason} -> - logger:log(error, "read_file_failed by ~p", [{Name, Reason, Position, Bytes}]), - return({error, Reason}) - end. - -%% this is an rpc call for stream_log_file/2 -read_trace_file(Name, Position, Limit) -> - TraceDir = emqx_mod_trace:trace_dir(), - {ok, AllFiles} = file:list_dir(TraceDir), - TracePrefix = "trace_" ++ binary_to_list(Name) ++ "_", - Filter = fun(FileName) -> nomatch =/= string:prefix(FileName, TracePrefix) end, - case lists:filter(Filter, AllFiles) of - [TraceFile] -> - TracePath = filename:join([TraceDir, TraceFile]), - read_file(TracePath, Position, Limit); - [] -> {error, not_found} - end. - -read_file(Path, Offset, Bytes) -> - {ok, IoDevice} = file:open(Path, [read, raw, binary]), - try - _ = case Offset of - 0 -> ok; - _ -> file:position(IoDevice, {bof, Offset}) - end, - file:read(IoDevice, Bytes) - after - file:close(IoDevice) - end. +stream_log_file(Path, Params) -> + return(emqx_trace_api:stream_log_file(Path, Params)). diff --git a/lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl deleted file mode 100644 index c643c7908..000000000 --- a/lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl +++ /dev/null @@ -1,478 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module(emqx_mod_trace_SUITE). - -%% API --compile(export_all). --compile(nowarn_export_all). - --include_lib("common_test/include/ct.hrl"). --include_lib("eunit/include/eunit.hrl"). --include_lib("emqx/include/emqx.hrl"). - --define(HOST, "http://127.0.0.1:18083/"). --define(API_VERSION, "v4"). --define(BASE_PATH, "api"). - --record(emqx_mod_trace, { - name, - type, - topic, - clientid, - packets = [], - enable = true, - start_at, - end_at, - log_size = #{} - }). - --define(PACKETS, ['CONNECT', 'CONNACK', 'PUBLISH', 'PUBACK', 'PUBREC', 'PUBREL' - , 'PUBCOMP', 'SUBSCRIBE', 'SUBACK', 'UNSUBSCRIBE', 'UNSUBACK' - , 'PINGREQ', 'PINGRESP', 'DISCONNECT', 'AUTH']). - -%%-------------------------------------------------------------------- -%% Setups -%%-------------------------------------------------------------------- - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_modules, emqx_dashboard]), - Config. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_modules, emqx_dashboard]). - -t_base_create_delete(_Config) -> - ok = emqx_mod_trace:clear(), - Now = erlang:system_time(second), - Start = to_rfc3339(Now), - End = to_rfc3339(Now + 30 * 60), - Name = <<"name1">>, - ClientId = <<"test-device">>, - Packets = [atom_to_binary(E) || E <- ?PACKETS], - Trace = [ - {<<"name">>, Name}, - {<<"type">>, <<"clientid">>}, - {<<"clientid">>, ClientId}, - {<<"packets">>, Packets}, - {<<"start_at">>, Start}, - {<<"end_at">>, End} - ], - AnotherTrace = lists:keyreplace(<<"name">>, 1, Trace, {<<"name">>, <<"AnotherName">>}), - ok = emqx_mod_trace:create(Trace), - ?assertEqual({error, {already_existed, Name}}, emqx_mod_trace:create(Trace)), - ?assertEqual({error, {duplicate_condition, Name}}, emqx_mod_trace:create(AnotherTrace)), - [TraceRec] = emqx_mod_trace:list(), - Expect = #emqx_mod_trace{ - name = Name, - type = clientid, - topic = undefined, - clientid = ClientId, - packets = ?PACKETS, - start_at = Now, - end_at = Now + 30 * 60 - }, - ?assertEqual(Expect, TraceRec), - ExpectFormat = [ - #{ - clientid => <<"test-device">>, - enable => true, - type => clientid, - packets => ?PACKETS, - name => <<"name1">>, - start_at => Start, - end_at => End, - log_size => #{}, - topic => undefined - } - ], - ?assertEqual(ExpectFormat, emqx_mod_trace:format([TraceRec])), - ?assertEqual(ok, emqx_mod_trace:delete(Name)), - ?assertEqual({error, not_found}, emqx_mod_trace:delete(Name)), - ?assertEqual([], emqx_mod_trace:list()), - ok. - -t_create_size_max(_Config) -> - emqx_mod_trace:clear(), - lists:map(fun(Seq) -> - Name = list_to_binary("name" ++ integer_to_list(Seq)), - Trace = [{<<"name">>, Name}, {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, - {<<"topic">>, list_to_binary("/x/y/" ++ integer_to_list(Seq))}], - ok = emqx_mod_trace:create(Trace) - end, lists:seq(1, 30)), - Trace31 = [{<<"name">>, <<"name31">>}, {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/31">>}], - {error, _} = emqx_mod_trace:create(Trace31), - ok = emqx_mod_trace:delete(<<"name30">>), - ok = emqx_mod_trace:create(Trace31), - ?assertEqual(30, erlang:length(emqx_mod_trace:list())), - ok. - -t_create_failed(_Config) -> - ok = emqx_mod_trace:clear(), - UnknownField = [{<<"unknown">>, 12}], - {error, Reason1} = emqx_mod_trace:create(UnknownField), - ?assertEqual(<<"unknown field: {<<\"unknown\">>,12}">>, iolist_to_binary(Reason1)), - - InvalidTopic = [{<<"topic">>, "#/#//"}], - {error, Reason2} = emqx_mod_trace:create(InvalidTopic), - ?assertEqual(<<"#/#// invalid by function_clause">>, iolist_to_binary(Reason2)), - - InvalidStart = [{<<"start_at">>, <<"2021-12-3:12">>}], - {error, Reason3} = emqx_mod_trace:create(InvalidStart), - ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, - iolist_to_binary(Reason3)), - - InvalidEnd = [{<<"end_at">>, <<"2021-12-3:12">>}], - {error, Reason4} = emqx_mod_trace:create(InvalidEnd), - ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, - iolist_to_binary(Reason4)), - - InvalidPackets = [{<<"packets">>, [<<"publish">>]}], - {error, Reason5} = emqx_mod_trace:create(InvalidPackets), - ?assertEqual(<<"unsupport packets: [publish]">>, iolist_to_binary(Reason5)), - - InvalidPackets2 = [{<<"packets">>, <<"publish">>}], - {error, Reason6} = emqx_mod_trace:create(InvalidPackets2), - ?assertEqual(<<"unsupport packets: <<\"publish\">>">>, iolist_to_binary(Reason6)), - - {error, Reason7} = emqx_mod_trace:create([{<<"name">>, <<"test">>}, {<<"type">>, <<"clientid">>}]), - ?assertEqual(<<"topic/clientid cannot be both empty">>, iolist_to_binary(Reason7)), - - InvalidPackets4 = [{<<"name">>, <<"/test">>}, {<<"clientid">>, <<"t">>}, {<<"type">>, <<"clientid">>}], - {error, Reason9} = emqx_mod_trace:create(InvalidPackets4), - ?assertEqual(<<"name cannot contain /">>, iolist_to_binary(Reason9)), - - ?assertEqual({error, "type required"}, emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, - {<<"packets">>, []}, {<<"clientid">>, <<"good">>}])), - - ?assertEqual({error, "incorrect type: only support clientid/topic"}, - emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, - {<<"packets">>, []}, {<<"clientid">>, <<"good">>}, {<<"type">>, <<"typeerror">> }])), - ok. - -t_create_default(_Config) -> - ok = emqx_mod_trace:clear(), - {error, "name required"} = emqx_mod_trace:create([]), - ok = emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, - {<<"type">>, <<"clientid">>}, {<<"packets">>, []}, {<<"clientid">>, <<"good">>}]), - [#emqx_mod_trace{packets = Packets}] = emqx_mod_trace:list(), - ?assertEqual([], Packets), - ok = emqx_mod_trace:clear(), - Trace = [ - {<<"name">>, <<"test-name">>}, - {<<"packets">>, [<<"PUBLISH">>]}, - {<<"type">>, <<"topic">>}, - {<<"topic">>, <<"/x/y/z">>}, - {<<"start_at">>, <<"2021-10-28T10:54:47+08:00">>}, - {<<"end_at">>, <<"2021-10-27T10:54:47+08:00">>} - ], - {error, "end_at time has already passed"} = emqx_mod_trace:create(Trace), - Now = erlang:system_time(second), - Trace2 = [ - {<<"name">>, <<"test-name">>}, - {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, - {<<"topic">>, <<"/x/y/z">>}, - {<<"start_at">>, to_rfc3339(Now + 10)}, - {<<"end_at">>, to_rfc3339(Now + 3)} - ], - {error, "failed by start_at >= end_at"} = emqx_mod_trace:create(Trace2), - ok = emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, - {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/z">>}]), - [#emqx_mod_trace{start_at = Start, end_at = End}] = emqx_mod_trace:list(), - ?assertEqual(10 * 60, End - Start), - ?assertEqual(true, Start - erlang:system_time(second) < 5), - ok. - -t_update_enable(_Config) -> - ok = emqx_mod_trace:clear(), - Name = <<"test-name">>, - Now = erlang:system_time(second), - End = list_to_binary(calendar:system_time_to_rfc3339(Now + 2)), - ok = emqx_mod_trace:create([{<<"name">>, Name}, {<<"packets">>, [<<"PUBLISH">>]}, - {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/z">>}, {<<"end_at">>, End}]), - [#emqx_mod_trace{enable = Enable}] = emqx_mod_trace:list(), - ?assertEqual(Enable, true), - ok = emqx_mod_trace:update(Name, false), - [#emqx_mod_trace{enable = false}] = emqx_mod_trace:list(), - ok = emqx_mod_trace:update(Name, false), - [#emqx_mod_trace{enable = false}] = emqx_mod_trace:list(), - ok = emqx_mod_trace:update(Name, true), - [#emqx_mod_trace{enable = true}] = emqx_mod_trace:list(), - ok = emqx_mod_trace:update(Name, false), - [#emqx_mod_trace{enable = false}] = emqx_mod_trace:list(), - ?assertEqual({error, not_found}, emqx_mod_trace:update(<<"Name not found">>, true)), - ct:sleep(2100), - ?assertEqual({error, finished}, emqx_mod_trace:update(Name, true)), - ok. - -t_load_state(_Config) -> - emqx_mod_trace:clear(), - ok = emqx_mod_trace:load(test), - Now = erlang:system_time(second), - Running = [{<<"name">>, <<"Running">>}, {<<"type">>, <<"topic">>}, - {<<"topic">>, <<"/x/y/1">>}, {<<"start_at">>, to_rfc3339(Now - 1)}, {<<"end_at">>, to_rfc3339(Now + 2)}], - Waiting = [{<<"name">>, <<"Waiting">>}, {<<"type">>, <<"topic">>}, - {<<"topic">>, <<"/x/y/2">>}, {<<"start_at">>, to_rfc3339(Now + 3)}, {<<"end_at">>, to_rfc3339(Now + 8)}], - Finished = [{<<"name">>, <<"Finished">>}, {<<"type">>, <<"topic">>}, - {<<"topic">>, <<"/x/y/3">>}, {<<"start_at">>, to_rfc3339(Now - 5)}, {<<"end_at">>, to_rfc3339(Now)}], - ok = emqx_mod_trace:create(Running), - ok = emqx_mod_trace:create(Waiting), - {error, "end_at time has already passed"} = emqx_mod_trace:create(Finished), - Traces = emqx_mod_trace:format(emqx_mod_trace:list()), - ?assertEqual(2, erlang:length(Traces)), - Enables = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces), - ExpectEnables = [{<<"Running">>, true}, {<<"Waiting">>, true}], - ?assertEqual(ExpectEnables, lists:sort(Enables)), - ct:sleep(3500), - Traces2 = emqx_mod_trace:format(emqx_mod_trace:list()), - ?assertEqual(2, erlang:length(Traces2)), - Enables2 = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces2), - ExpectEnables2 = [{<<"Running">>, false}, {<<"Waiting">>, true}], - ?assertEqual(ExpectEnables2, lists:sort(Enables2)), - ok = emqx_mod_trace:unload(test), - ok. - -t_client_event(_Config) -> - application:set_env(emqx, allow_anonymous, true), - emqx_mod_trace:clear(), - ClientId = <<"client-test">>, - ok = emqx_mod_trace:load(test), - Now = erlang:system_time(second), - Start = to_rfc3339(Now), - Name = <<"test_client_id_event">>, - ok = emqx_mod_trace:create([{<<"name">>, Name}, - {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), - ct:sleep(200), - {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), - {ok, _} = emqtt:connect(Client), - emqtt:ping(Client), - ok = emqtt:publish(Client, <<"/test">>, #{}, <<"1">>, [{qos, 0}]), - ok = emqtt:publish(Client, <<"/test">>, #{}, <<"2">>, [{qos, 0}]), - ct:sleep(200), - ok = emqx_mod_trace:create([{<<"name">>, <<"test_topic">>}, - {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/test">>}, {<<"start_at">>, Start}]), - ct:sleep(200), - {ok, Bin} = file:read_file(emqx_mod_trace:log_file(Name, Now)), - ok = emqtt:publish(Client, <<"/test">>, #{}, <<"3">>, [{qos, 0}]), - ok = emqtt:publish(Client, <<"/test">>, #{}, <<"4">>, [{qos, 0}]), - ok = emqtt:disconnect(Client), - ct:sleep(200), - {ok, Bin2} = file:read_file(emqx_mod_trace:log_file(Name, Now)), - {ok, Bin3} = file:read_file(emqx_mod_trace:log_file(<<"test_topic">>, Now)), - ct:pal("Bin ~p Bin2 ~p Bin3 ~p", [byte_size(Bin), byte_size(Bin2), byte_size(Bin3)]), - ?assert(erlang:byte_size(Bin) > 0), - ?assert(erlang:byte_size(Bin) < erlang:byte_size(Bin2)), - ?assert(erlang:byte_size(Bin3) > 0), - ok = emqx_mod_trace:unload(test), - ok. - -t_get_log_filename(_Config) -> - ok = emqx_mod_trace:clear(), - ok = emqx_mod_trace:load(test), - Now = erlang:system_time(second), - Start = calendar:system_time_to_rfc3339(Now), - End = calendar:system_time_to_rfc3339(Now + 2), - Name = <<"name1">>, - ClientId = <<"test-device">>, - Packets = [atom_to_binary(E) || E <- ?PACKETS], - Trace = [ - {<<"name">>, Name}, - {<<"type">>, <<"clientid">>}, - {<<"clientid">>, ClientId}, - {<<"packets">>, Packets}, - {<<"start_at">>, list_to_binary(Start)}, - {<<"end_at">>, list_to_binary(End)} - ], - ok = emqx_mod_trace:create(Trace), - ?assertEqual({error, not_found}, emqx_mod_trace:get_trace_filename(<<"test">>)), - ?assertEqual(ok, element(1, emqx_mod_trace:get_trace_filename(Name))), - ct:sleep(3000), - ?assertEqual(ok, element(1, emqx_mod_trace:get_trace_filename(Name))), - ok = emqx_mod_trace:unload(test), - ok. - -t_trace_file(_Config) -> - FileName = "test.log", - Content = <<"test \n test">>, - TraceDir = emqx_mod_trace:trace_dir(), - File = filename:join(TraceDir, FileName), - ok = file:write_file(File, Content), - {ok, Node, Bin} = emqx_mod_trace:trace_file(FileName), - ?assertEqual(Node, atom_to_list(node())), - ?assertEqual(Content, Bin), - ok = file:delete(File), - ok. - -t_http_test(_Config) -> - emqx_mod_trace:clear(), - emqx_mod_trace:load(test), - Header = auth_header_(), - %% list - {ok, Empty} = request_api(get, api_path("trace"), Header), - ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(Empty)), - %% create - ErrorTrace = #{}, - {ok, Error} = request_api(post, api_path("trace"), Header, ErrorTrace), - ?assertEqual(#{<<"message">> => <<"unknown field: {}">>, <<"code">> => <<"INCORRECT_PARAMS">>}, json(Error)), - - Name = <<"test-name">>, - Trace = [ - {<<"name">>, Name}, - {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, - {<<"topic">>, <<"/x/y/z">>} - ], - - {ok, Create} = request_api(post, api_path("trace"), Header, Trace), - ?assertEqual(#{<<"code">> => 0}, json(Create)), - - {ok, List} = request_api(get, api_path("trace"), Header), - #{<<"code">> := 0, <<"data">> := [Data]} = json(List), - ?assertEqual(Name, maps:get(<<"name">>, Data)), - - %% update - {ok, Update} = request_api(put, api_path("trace/test-name/disable"), Header, #{}), - ?assertEqual(#{<<"code">> => 0, - <<"data">> => #{<<"enable">> => false, - <<"name">> => <<"test-name">>}}, json(Update)), - - {ok, List1} = request_api(get, api_path("trace"), Header), - #{<<"code">> := 0, <<"data">> := [Data1]} = json(List1), - ?assertEqual(false, maps:get(<<"enable">>, Data1)), - - %% delete - {ok, Delete} = request_api(delete, api_path("trace/test-name"), Header), - ?assertEqual(#{<<"code">> => 0}, json(Delete)), - - {ok, DeleteNotFound} = request_api(delete, api_path("trace/test-name"), Header), - ?assertEqual(#{<<"code">> => <<"NOT_FOUND">>, - <<"message">> => <<"test-nameNOT FOUND">>}, json(DeleteNotFound)), - - {ok, List2} = request_api(get, api_path("trace"), Header), - ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(List2)), - - %% clear - {ok, Create1} = request_api(post, api_path("trace"), Header, Trace), - ?assertEqual(#{<<"code">> => 0}, json(Create1)), - - {ok, Clear} = request_api(delete, api_path("trace"), Header), - ?assertEqual(#{<<"code">> => 0}, json(Clear)), - - emqx_mod_trace:unload(test), - ok. - -t_download_log(_Config) -> - emqx_mod_trace:clear(), - emqx_mod_trace:load(test), - ClientId = <<"client-test">>, - Now = erlang:system_time(second), - Start = to_rfc3339(Now), - Name = <<"test_client_id">>, - ok = emqx_mod_trace:create([{<<"name">>, Name}, - {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), - {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), - {ok, _} = emqtt:connect(Client), - [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], - ct:sleep(100), - {ok, #{}, {sendfile, 0, ZipFileSize, _ZipFile}} = - emqx_mod_trace_api:download_zip_log(#{name => Name}, []), - ?assert(ZipFileSize > 0), - %% download zip file failed by server_closed occasionally? - %Header = auth_header_(), - %{ok, ZipBin} = request_api(get, api_path("trace/test_client_id/download"), Header), - %{ok, ZipHandler} = zip:zip_open(ZipBin), - %{ok, [ZipName]} = zip:zip_get(ZipHandler), - %?assertNotEqual(nomatch, string:find(ZipName, "test@127.0.0.1")), - %{ok, _} = file:read_file(emqx_mod_trace:log_file(<<"test_client_id">>, Now)), - ok = emqtt:disconnect(Client), - emqx_mod_trace:unload(test), - ok. - -t_stream_log(_Config) -> - application:set_env(emqx, allow_anonymous, true), - emqx_mod_trace:clear(), - emqx_mod_trace:load(test), - ClientId = <<"client-stream">>, - Now = erlang:system_time(second), - Name = <<"test_stream_log">>, - Start = to_rfc3339(Now - 10), - ok = emqx_mod_trace:create([{<<"name">>, Name}, - {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), - ct:sleep(200), - {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), - {ok, _} = emqtt:connect(Client), - [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], - emqtt:publish(Client, <<"/good">>, #{}, <<"ghood1">>, [{qos, 0}]), - emqtt:publish(Client, <<"/good">>, #{}, <<"ghood2">>, [{qos, 0}]), - ok = emqtt:disconnect(Client), - ct:sleep(200), - File = emqx_mod_trace:log_file(Name, Now), - ct:pal("FileName: ~p", [File]), - {ok, FileBin} = file:read_file(File), - ct:pal("FileBin: ~p ~s", [byte_size(FileBin), FileBin]), - Header = auth_header_(), - {ok, Binary} = request_api(get, api_path("trace/test_stream_log/log?_limit=10"), Header), - #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta, <<"items">> := Bin}} = json(Binary), - ?assertEqual(10, byte_size(Bin)), - ?assertEqual(#{<<"page">> => 10, <<"limit">> => 10}, Meta), - {ok, Binary1} = request_api(get, api_path("trace/test_stream_log/log?_page=20&_limit=10"), Header), - #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta1, <<"items">> := Bin1}} = json(Binary1), - ?assertEqual(#{<<"page">> => 30, <<"limit">> => 10}, Meta1), - ?assertEqual(10, byte_size(Bin1)), - emqx_mod_trace:unload(test), - ok. - -to_rfc3339(Second) -> - list_to_binary(calendar:system_time_to_rfc3339(Second)). - -auth_header_() -> - auth_header_("admin", "public"). - -auth_header_(User, Pass) -> - Encoded = base64:encode_to_string(lists:append([User, ":", Pass])), - {"Authorization", "Basic " ++ Encoded}. - -request_api(Method, Url, Auth) -> do_request_api(Method, {Url, [Auth]}). - -request_api(Method, Url, Auth, Body) -> - Request = {Url, [Auth], "application/json", emqx_json:encode(Body)}, - do_request_api(Method, Request). - -do_request_api(Method, Request) -> - ct:pal("Method: ~p, Request: ~p", [Method, Request]), - case httpc:request(Method, Request, [], [{body_format, binary}]) of - {error, socket_closed_remotely} -> - {error, socket_closed_remotely}; - {error,{shutdown, server_closed}} -> - {error, server_closed}; - {ok, {{"HTTP/1.1", Code, _}, _Headers, Return} } - when Code =:= 200 orelse Code =:= 201 -> - {ok, Return}; - {ok, {Reason, _, _}} -> - {error, Reason} - end. - -api_path(Path) -> - ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION, Path]). - -json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), Jsx. diff --git a/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl new file mode 100644 index 000000000..609a2d93c --- /dev/null +++ b/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl @@ -0,0 +1,179 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mod_trace_api_SUITE). + +%% API +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-define(HOST, "http://127.0.0.1:18083/"). +-define(API_VERSION, "v4"). +-define(BASE_PATH, "api"). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx_plugin_libs), + emqx_ct_helpers:start_apps([emqx_modules, emqx_dashboard]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_modules, emqx_dashboard]). + +t_http_test(_Config) -> + emqx_trace:clear(), + load(), + Header = auth_header_(), + %% list + {ok, Empty} = request_api(get, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(Empty)), + %% create + ErrorTrace = #{}, + {ok, Error} = request_api(post, api_path("trace"), Header, ErrorTrace), + ?assertEqual(#{<<"message">> => <<"unknown field: {}">>, + <<"code">> => <<"INCORRECT_PARAMS">>}, json(Error)), + + Name = <<"test-name">>, + Trace = [ + {<<"name">>, Name}, + {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"topic">>, <<"/x/y/z">>} + ], + + {ok, Create} = request_api(post, api_path("trace"), Header, Trace), + ?assertEqual(#{<<"code">> => 0}, json(Create)), + + {ok, List} = request_api(get, api_path("trace"), Header), + #{<<"code">> := 0, <<"data">> := [Data]} = json(List), + ?assertEqual(Name, maps:get(<<"name">>, Data)), + + %% update + {ok, Update} = request_api(put, api_path("trace/test-name/disable"), Header, #{}), + ?assertEqual(#{<<"code">> => 0, + <<"data">> => #{<<"enable">> => false, + <<"name">> => <<"test-name">>}}, json(Update)), + + {ok, List1} = request_api(get, api_path("trace"), Header), + #{<<"code">> := 0, <<"data">> := [Data1]} = json(List1), + ?assertEqual(false, maps:get(<<"enable">>, Data1)), + + %% delete + {ok, Delete} = request_api(delete, api_path("trace/test-name"), Header), + ?assertEqual(#{<<"code">> => 0}, json(Delete)), + + {ok, DeleteNotFound} = request_api(delete, api_path("trace/test-name"), Header), + ?assertEqual(#{<<"code">> => <<"NOT_FOUND">>, + <<"message">> => <<"test-nameNOT FOUND">>}, json(DeleteNotFound)), + + {ok, List2} = request_api(get, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(List2)), + + %% clear + {ok, Create1} = request_api(post, api_path("trace"), Header, Trace), + ?assertEqual(#{<<"code">> => 0}, json(Create1)), + + {ok, Clear} = request_api(delete, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0}, json(Clear)), + + unload(), + ok. + +t_stream_log(_Config) -> + application:set_env(emqx, allow_anonymous, true), + emqx_trace:clear(), + load(), + ClientId = <<"client-stream">>, + Now = erlang:system_time(second), + Name = <<"test_stream_log">>, + Start = to_rfc3339(Now - 10), + ok = emqx_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + ct:sleep(200), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], + emqtt:publish(Client, <<"/good">>, #{}, <<"ghood1">>, [{qos, 0}]), + emqtt:publish(Client, <<"/good">>, #{}, <<"ghood2">>, [{qos, 0}]), + ok = emqtt:disconnect(Client), + ct:sleep(200), + File = emqx_trace:log_file(Name, Now), + ct:pal("FileName: ~p", [File]), + {ok, FileBin} = file:read_file(File), + ct:pal("FileBin: ~p ~s", [byte_size(FileBin), FileBin]), + Header = auth_header_(), + {ok, Binary} = request_api(get, api_path("trace/test_stream_log/log?_limit=10"), Header), + #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta, <<"items">> := Bin}} = json(Binary), + ?assertEqual(10, byte_size(Bin)), + ?assertEqual(#{<<"page">> => 10, <<"limit">> => 10}, Meta), + Path = api_path("trace/test_stream_log/log?_page=20&_limit=10"), + {ok, Binary1} = request_api(get, Path, Header), + #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta1, <<"items">> := Bin1}} = json(Binary1), + ?assertEqual(#{<<"page">> => 30, <<"limit">> => 10}, Meta1), + ?assertEqual(10, byte_size(Bin1)), + unload(), + ok. + +to_rfc3339(Second) -> + list_to_binary(calendar:system_time_to_rfc3339(Second)). + +auth_header_() -> + auth_header_("admin", "public"). + +auth_header_(User, Pass) -> + Encoded = base64:encode_to_string(lists:append([User, ":", Pass])), + {"Authorization", "Basic " ++ Encoded}. + +request_api(Method, Url, Auth) -> do_request_api(Method, {Url, [Auth]}). + +request_api(Method, Url, Auth, Body) -> + Request = {Url, [Auth], "application/json", emqx_json:encode(Body)}, + do_request_api(Method, Request). + +do_request_api(Method, Request) -> + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], [{body_format, binary}]) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {error,{shutdown, server_closed}} -> + {error, server_closed}; + {ok, {{"HTTP/1.1", Code, _}, _Headers, Return} } + when Code =:= 200 orelse Code =:= 201 -> + {ok, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +api_path(Path) -> + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION, Path]). + +json(Data) -> + {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), Jsx. + +load() -> + emqx_trace:start_link(). + +unload() -> + gen_server:stop(emqx_trace). From a4a7cac647909116ea0b2094b97ab0917c7755e5 Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 12 Nov 2021 14:00:48 +0800 Subject: [PATCH 018/104] fix(eqmx_st_statistics): add ignore_before_create in config (#6140) 1. allows not to process the message before the session is created to solve the problem caused by clean session = false 2. fix some elvis errors --- .../emqx_st_statistics/emqx_st_statistics.erl | 27 ++++++++---- etc/emqx.conf | 5 +++ priv/emqx.schema | 5 +++ src/emqx_session.erl | 43 ++++++++++--------- 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl index ac1604c9b..21cf2692e 100644 --- a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl +++ b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl @@ -20,11 +20,11 @@ -include_lib("include/emqx.hrl"). -include_lib("include/logger.hrl"). --include("include/emqx_st_statistics.hrl"). +-include_lib("emqx_plugin_libs/include/emqx_st_statistics.hrl"). -logger_header("[SLOW TOPICS]"). --export([ start_link/1, on_publish_done/3, enable/0 +-export([ start_link/1, on_publish_done/5, enable/0 , disable/0, clear_history/0 ]). @@ -90,8 +90,13 @@ start_link(Env) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). --spec on_publish_done(message(), pos_integer(), counters:counters_ref()) -> ok. -on_publish_done(#message{timestamp = Timestamp} = Msg, Threshold, Counter) -> +-spec on_publish_done(message(), + pos_integer(), boolean(), pos_integer(), counters:counters_ref()) -> ok. +on_publish_done(#message{timestamp = Timestamp}, Created, IgnoreBeforeCreate, _, _) + when IgnoreBeforeCreate, Timestamp < Created -> + ok; + +on_publish_done(#message{timestamp = Timestamp} = Msg, _, _, Threshold, Counter) -> case ?NOW - Timestamp of Elapsed when Elapsed > Threshold -> case get_log_quota(Counter) of @@ -125,7 +130,8 @@ init([Env]) -> Counter = counters:new(1, [write_concurrency]), set_log_quota(Env, Counter), Threshold = get_value(threshold_time, Env), - load(Threshold, Counter), + IgnoreBeforeCreate = get_value(ignore_before_create, Env), + load(IgnoreBeforeCreate, Threshold, Counter), {ok, #{config => Env, period => 1, last_tick_at => ?NOW, @@ -139,7 +145,8 @@ handle_call({enable, Enable}, _From, State; true -> Threshold = get_value(threshold_time, Cfg), - load(Threshold, Counter), + IgnoreBeforeCreate = get_value(ignore_before_create, Cfg), + load(IgnoreBeforeCreate, Threshold, Counter), State#{enable := true}; _ -> unload(), @@ -319,12 +326,14 @@ publish(TickTime, Cfg, Notices) -> }), ok. -load(Threshold, Counter) -> - _ = emqx:hook('message.publish_done', fun ?MODULE:on_publish_done/3, [Threshold, Counter]), +load(IgnoreBeforeCreate, Threshold, Counter) -> + _ = emqx:hook('message.publish_done', + fun ?MODULE:on_publish_done/5, + [IgnoreBeforeCreate, Threshold, Counter]), ok. unload() -> - emqx:unhook('message.publish_done', fun ?MODULE:on_publish_done/3). + emqx:unhook('message.publish_done', fun ?MODULE:on_publish_done/5). -spec get_topic(proplists:proplist()) -> binary(). get_topic(Cfg) -> diff --git a/etc/emqx.conf b/etc/emqx.conf index a94afd961..7b81e40cc 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -2219,6 +2219,11 @@ module.presence.qos = 1 ## Default: 10 seconds #module.st_statistics.threshold_time = 10S +## ignore the messages that before than session created +## +## Default: true +#module.st_statistics.ignore_before_create = true + ## Time window of slow topics statistics ## ## Value: 5 minutes diff --git a/priv/emqx.schema b/priv/emqx.schema index 69f3acc41..602de56b9 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -2193,6 +2193,11 @@ end}. {datatype, {duration, ms}} ]}. +{mapping, "module.st_statistics.ignore_before_create", "emqx.modules", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + {mapping, "module.st_statistics.time_window", "emqx.modules", [ {default, "5M"}, {datatype, {duration, ms}} diff --git a/src/emqx_session.erl b/src/emqx_session.erl index db09bf8c9..d4b671e71 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -269,7 +269,8 @@ unsubscribe(ClientInfo, TopicFilter, UnSubOpts, Session = #session{subscriptions case maps:find(TopicFilter, Subs) of {ok, SubOpts} -> ok = emqx_broker:unsubscribe(TopicFilter), - ok = emqx_hooks:run('session.unsubscribed', [ClientInfo, TopicFilter, maps:merge(SubOpts, UnSubOpts)]), + ok = emqx_hooks:run('session.unsubscribed', + [ClientInfo, TopicFilter, maps:merge(SubOpts, UnSubOpts)]), {ok, Session#session{subscriptions = maps:remove(TopicFilter, Subs)}}; error -> {error, ?RC_NO_SUBSCRIPTION_EXISTED} @@ -316,10 +317,10 @@ is_awaiting_full(#session{awaiting_rel = AwaitingRel, -> {ok, emqx_types:message(), session()} | {ok, emqx_types:message(), replies(), session()} | {error, emqx_types:reason_code()}). -puback(PacketId, Session = #session{inflight = Inflight}) -> +puback(PacketId, Session = #session{inflight = Inflight, created_at = CreatedAt}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> - emqx:run_hook('message.publish_done', [Msg]), + emqx:run_hook('message.publish_done', [Msg, CreatedAt]), Inflight1 = emqx_inflight:delete(PacketId, Inflight), return_with(Msg, dequeue(Session#session{inflight = Inflight1})); {value, {_Pubrel, _Ts}} -> @@ -341,11 +342,11 @@ return_with(Msg, {ok, Publishes, Session}) -> -spec(pubrec(emqx_types:packet_id(), session()) -> {ok, emqx_types:message(), session()} | {error, emqx_types:reason_code()}). -pubrec(PacketId, Session = #session{inflight = Inflight}) -> +pubrec(PacketId, Session = #session{inflight = Inflight, created_at = CreatedAt}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> %% execute hook here, because message record will be replaced by pubrel - emqx:run_hook('message.publish_done', [Msg]), + emqx:run_hook('message.publish_done', [Msg, CreatedAt]), Inflight1 = emqx_inflight:update(PacketId, with_ts(pubrel), Inflight), {ok, Msg, Session#session{inflight = Inflight1}}; {value, {pubrel, _Ts}} -> @@ -408,7 +409,7 @@ dequeue(Cnt, Msgs, Q) -> case emqx_message:is_expired(Msg) of true -> ok = inc_expired_cnt(delivery), dequeue(Cnt, Msgs, Q1); - false -> dequeue(acc_cnt(Msg, Cnt), [Msg|Msgs], Q1) + false -> dequeue(acc_cnt(Msg, Cnt), [Msg | Msgs], Q1) end end. @@ -438,11 +439,11 @@ deliver([Msg | More], Acc, Session) -> {ok, Session1} -> deliver(More, Acc, Session1); {ok, [Publish], Session1} -> - deliver(More, [Publish|Acc], Session1) + deliver(More, [Publish | Acc], Session1) end. deliver_msg(Msg = #message{qos = ?QOS_0}, Session) -> - emqx:run_hook('message.publish_done', [Msg]), + emqx:run_hook('message.publish_done', [Msg, Session#session.created_at]), {ok, [{undefined, maybe_ack(Msg)}], Session}; deliver_msg(Msg = #message{qos = QoS}, Session = @@ -461,7 +462,7 @@ deliver_msg(Msg = #message{qos = QoS}, Session = {ok, [Publish], next_pkt_id(Session1)} end. --spec(enqueue(list(emqx_types:deliver())|emqx_types:message(), +-spec(enqueue(list(emqx_types:deliver()) | emqx_types:message(), session()) -> session()). enqueue([Deliver], Session) -> %% Optimize Enrich = enrich_fun(Session), @@ -512,23 +513,23 @@ get_subopts(Topic, SubMap) -> end. enrich_subopts([], Msg, _Session) -> Msg; -enrich_subopts([{nl, 1}|Opts], Msg, Session) -> +enrich_subopts([{nl, 1} | Opts], Msg, Session) -> enrich_subopts(Opts, emqx_message:set_flag(nl, Msg), Session); -enrich_subopts([{nl, 0}|Opts], Msg, Session) -> +enrich_subopts([{nl, 0} | Opts], Msg, Session) -> enrich_subopts(Opts, Msg, Session); -enrich_subopts([{qos, SubQoS}|Opts], Msg = #message{qos = PubQoS}, +enrich_subopts([{qos, SubQoS} | Opts], Msg = #message{qos = PubQoS}, Session = #session{upgrade_qos = true}) -> enrich_subopts(Opts, Msg#message{qos = max(SubQoS, PubQoS)}, Session); -enrich_subopts([{qos, SubQoS}|Opts], Msg = #message{qos = PubQoS}, +enrich_subopts([{qos, SubQoS} | Opts], Msg = #message{qos = PubQoS}, Session = #session{upgrade_qos = false}) -> enrich_subopts(Opts, Msg#message{qos = min(SubQoS, PubQoS)}, Session); -enrich_subopts([{rap, 1}|Opts], Msg, Session) -> +enrich_subopts([{rap, 1} | Opts], Msg, Session) -> enrich_subopts(Opts, Msg, Session); -enrich_subopts([{rap, 0}|Opts], Msg = #message{headers = #{retained := true}}, Session) -> +enrich_subopts([{rap, 0} | Opts], Msg = #message{headers = #{retained := true}}, Session) -> enrich_subopts(Opts, Msg, Session); -enrich_subopts([{rap, 0}|Opts], Msg, Session) -> +enrich_subopts([{rap, 0} | Opts], Msg, Session) -> enrich_subopts(Opts, emqx_message:set_flag(retain, false, Msg), Session); -enrich_subopts([{subid, SubId}|Opts], Msg, Session) -> +enrich_subopts([{subid, SubId} | Opts], Msg, Session) -> Props = emqx_message:get_header(properties, Msg, #{}), Msg1 = emqx_message:set_header(properties, Props#{'Subscription-Identifier' => SubId}, Msg), enrich_subopts(Opts, Msg1, Session). @@ -556,8 +557,8 @@ retry(Session = #session{inflight = Inflight}) -> retry_delivery([], Acc, _Now, Session = #session{retry_interval = Interval}) -> {ok, lists:reverse(Acc), Interval, Session}; -retry_delivery([{PacketId, {Msg, Ts}}|More], Acc, Now, Session = - #session{retry_interval = Interval, inflight = Inflight}) -> +retry_delivery([{PacketId, {Msg, Ts}} | More], Acc, Now, Session = + #session{retry_interval = Interval, inflight = Inflight}) -> case (Age = age(Now, Ts)) >= Interval of true -> {Acc1, Inflight1} = retry_delivery(PacketId, Msg, Now, Acc, Inflight), @@ -574,12 +575,12 @@ retry_delivery(PacketId, Msg, Now, Acc, Inflight) when is_record(Msg, message) - false -> Msg1 = emqx_message:set_flag(dup, true, Msg), Inflight1 = emqx_inflight:update(PacketId, {Msg1, Now}, Inflight), - {[{PacketId, Msg1}|Acc], Inflight1} + {[{PacketId, Msg1} | Acc], Inflight1} end; retry_delivery(PacketId, pubrel, Now, Acc, Inflight) -> Inflight1 = emqx_inflight:update(PacketId, {pubrel, Now}, Inflight), - {[{pubrel, PacketId}|Acc], Inflight1}. + {[{pubrel, PacketId} | Acc], Inflight1}. %%-------------------------------------------------------------------- %% Expire Awaiting Rel From e7bbe98a7a6a095a7d8ee1fea84d424012717d3c Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 12 Nov 2021 13:53:34 +0800 Subject: [PATCH 019/104] fix(ekka): update ekka to 0.8.2 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index a38d74e88..cc147dec3 100644 --- a/rebar.config +++ b/rebar.config @@ -43,7 +43,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.4"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.2"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.6.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}} From 66d0c44e3612a3446970b121f30efbac57b78abe Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 12 Nov 2021 14:51:02 +0800 Subject: [PATCH 020/104] fix(emqx_retainer): refresh the timestamp when dispatch retained message (#6148) --- apps/emqx_retainer/src/emqx_retainer.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 340e6929d..8884731f3 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -78,7 +78,8 @@ dispatch(Pid, Topic) -> false -> read_messages(Topic); true -> match_messages(Topic) end, - [Pid ! {deliver, Topic, Msg} || Msg <- sort_retained(Msgs)]. + Now = erlang:system_time(millisecond), + [Pid ! {deliver, Topic, Msg#message{timestamp = Now}} || Msg <- sort_retained(Msgs)]. %% RETAIN flag set to 1 and payload containing zero bytes on_message_publish(Msg = #message{flags = #{retain := true}, From f3de1bdb7715e828cdb185c272f74790f3b8b24b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 12 Nov 2021 15:48:30 +0800 Subject: [PATCH 021/104] fix: make sure enable boolean() (#6147) --- apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index 1bc71654e..a5e5b2906 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -34,7 +34,7 @@ list_trace(_, Params) -> List = case Params of - [{<<"enable">>, Enable}] -> emqx_trace:list(Enable); + [{<<"enable">>, Enable}] -> emqx_trace:list(binary_to_existing_atom(Enable)); _ -> emqx_trace:list() end, {ok, emqx_trace:format(List)}. From e9ce8e7586143175590e5805c6c30ec15e04270b Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 12 Nov 2021 18:31:55 +0800 Subject: [PATCH 022/104] fix(emqx_retainer): revert the "refresh the timestamp when dispatch retained message" This reverts commit 66d0c44e3612a3446970b121f30efbac57b78abe. --- apps/emqx_retainer/src/emqx_retainer.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 8884731f3..340e6929d 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -78,8 +78,7 @@ dispatch(Pid, Topic) -> false -> read_messages(Topic); true -> match_messages(Topic) end, - Now = erlang:system_time(millisecond), - [Pid ! {deliver, Topic, Msg#message{timestamp = Now}} || Msg <- sort_retained(Msgs)]. + [Pid ! {deliver, Topic, Msg} || Msg <- sort_retained(Msgs)]. %% RETAIN flag set to 1 and payload containing zero bytes on_message_publish(Msg = #message{flags = #{retain := true}, From 8bf6668e4c0475fadea06ef79e0b4e068bd40971 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 12 Nov 2021 20:11:59 +0800 Subject: [PATCH 023/104] Set keepalive via http api (#6143) * feat: set keepalive over http api * chore: elvis warning * fix: bump retainer to 4.4.0 --- apps/emqx_management/src/emqx_mgmt.erl | 47 +++--- .../src/emqx_mgmt_api_clients.erl | 58 +++++-- .../test/emqx_mgmt_api_SUITE.erl | 143 +++++++++++++----- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- src/emqx_channel.erl | 23 ++- src/emqx_keepalive.erl | 8 +- 6 files changed, 200 insertions(+), 81 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index bfba34603..93f24cd37 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -22,6 +22,9 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_mgmt]}}]). +-elvis([{elvis_style, god_modules, #{ignore => [emqx_mgmt]}}]). + %% Nodes and Brokers API -export([ list_nodes/0 , lookup_node/1 @@ -49,6 +52,7 @@ , clean_acl_cache_all/1 , set_ratelimit_policy/2 , set_quota_policy/2 + , set_keepalive/2 ]). %% Internal funcs @@ -142,7 +146,8 @@ node_info(Node) when Node =:= node() -> memory_used => proplists:get_value(total, Memory), process_available => erlang:system_info(process_limit), process_used => erlang:system_info(process_count), - max_fds => proplists:get_value(max_fds, lists:usort(lists:flatten(erlang:system_info(check_io)))), + max_fds => proplists:get_value(max_fds, + lists:usort(lists:flatten(erlang:system_info(check_io)))), connections => ets:info(emqx_channel, size), node_status => 'Running', uptime => iolist_to_binary(proplists:get_value(uptime, BrokerInfo)), @@ -196,10 +201,12 @@ get_stats(Node) -> %%-------------------------------------------------------------------- lookup_client({clientid, ClientId}, FormatFun) -> - lists:append([lookup_client(Node, {clientid, ClientId}, FormatFun) || Node <- ekka_mnesia:running_nodes()]); + lists:append([lookup_client(Node, {clientid, ClientId}, FormatFun) + || Node <- ekka_mnesia:running_nodes()]); lookup_client({username, Username}, FormatFun) -> - lists:append([lookup_client(Node, {username, Username}, FormatFun) || Node <- ekka_mnesia:running_nodes()]). + lists:append([lookup_client(Node, {username, Username}, FormatFun) + || Node <- ekka_mnesia:running_nodes()]). lookup_client(Node, {clientid, ClientId}, {M,F}) when Node =:= node() -> lists:append(lists:map( @@ -222,10 +229,7 @@ lookup_client(Node, {username, Username}, FormatFun) -> kickout_client(ClientId) -> Results = [kickout_client(Node, ClientId) || Node <- ekka_mnesia:running_nodes()], - case lists:any(fun(Item) -> Item =:= ok end, Results) of - true -> ok; - false -> lists:last(Results) - end. + has_any_ok(Results). kickout_client(Node, ClientId) when Node =:= node() -> emqx_cm:kick_session(ClientId); @@ -238,10 +242,7 @@ list_acl_cache(ClientId) -> clean_acl_cache(ClientId) -> Results = [clean_acl_cache(Node, ClientId) || Node <- ekka_mnesia:running_nodes()], - case lists:any(fun(Item) -> Item =:= ok end, Results) of - true -> ok; - false -> lists:last(Results) - end. + has_any_ok(Results). clean_acl_cache(Node, ClientId) when Node =:= node() -> case emqx_cm:lookup_channels(ClientId) of @@ -273,6 +274,9 @@ set_ratelimit_policy(ClientId, Policy) -> set_quota_policy(ClientId, Policy) -> call_client(ClientId, {quota, Policy}). +set_keepalive(ClientId, Interval) -> + call_client(ClientId, {keepalive, Interval}). + %% @private call_client(ClientId, Req) -> Results = [call_client(Node, ClientId, Req) || Node <- ekka_mnesia:running_nodes()], @@ -281,7 +285,7 @@ call_client(ClientId, Req) -> end, Results), case Expected of [] -> {error, not_found}; - [Result|_] -> Result + [Result | _] -> Result end. %% @private @@ -313,7 +317,8 @@ list_subscriptions(Node) -> rpc_call(Node, list_subscriptions, [Node]). list_subscriptions_via_topic(Topic, FormatFun) -> - lists:append([list_subscriptions_via_topic(Node, Topic, FormatFun) || Node <- ekka_mnesia:running_nodes()]). + lists:append([list_subscriptions_via_topic(Node, Topic, FormatFun) + || Node <- ekka_mnesia:running_nodes()]). list_subscriptions_via_topic(Node, Topic, {M,F}) when Node =:= node() -> MatchSpec = [{{{'_', '$1'}, '_'}, [{'=:=','$1', Topic}], ['$_']}], @@ -436,7 +441,8 @@ list_listeners(Node) when Node =:= node() -> Http = lists:map(fun({Protocol, Opts}) -> #{protocol => Protocol, listen_on => proplists:get_value(port, Opts), - acceptors => maps:get(num_acceptors, proplists:get_value(transport_options, Opts, #{}), 0), + acceptors => maps:get(num_acceptors, + proplists:get_value(transport_options, Opts, #{}), 0), max_conns => proplists:get_value(max_connections, Opts), current_conns => proplists:get_value(all_connections, Opts), shutdown_count => []} @@ -483,9 +489,10 @@ add_duration_field(Alarms) -> add_duration_field([], _Now, Acc) -> Acc; -add_duration_field([Alarm = #{activated := true, activate_at := ActivateAt}| Rest], Now, Acc) -> +add_duration_field([Alarm = #{activated := true, activate_at := ActivateAt} | Rest], Now, Acc) -> add_duration_field(Rest, Now, [Alarm#{duration => Now - ActivateAt} | Acc]); -add_duration_field([Alarm = #{activated := false, activate_at := ActivateAt, deactivate_at := DeactivateAt}| Rest], Now, Acc) -> +add_duration_field([Alarm = #{activated := false, + activate_at := ActivateAt, deactivate_at := DeactivateAt} | Rest], Now, Acc) -> add_duration_field(Rest, Now, [Alarm#{duration => DeactivateAt - ActivateAt} | Acc]). %%-------------------------------------------------------------------- @@ -560,7 +567,7 @@ check_row_limit(Tables) -> check_row_limit([], _Limit) -> ok; -check_row_limit([Tab|Tables], Limit) -> +check_row_limit([Tab | Tables], Limit) -> case table_size(Tab) > Limit of true -> false; false -> check_row_limit(Tables, Limit) @@ -571,4 +578,8 @@ max_row_limit() -> table_size(Tab) -> ets:info(Tab, size). - +has_any_ok(Results) -> + case lists:any(fun(Item) -> Item =:= ok end, Results) of + true -> ok; + false -> lists:last(Results) + end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 1ddd87a3d..5aacc7895 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -117,6 +117,12 @@ func => clean_quota, descr => "Clear the quota policy"}). +-rest_api(#{name => set_keepalive, + method => 'PUT', + path => "/clients/:bin:clientid/keepalive", + func => set_keepalive, + descr => "Set the client keepalive"}). + -import(emqx_mgmt_util, [ ntoa/1 , strftime/1 ]). @@ -130,23 +136,24 @@ , set_quota_policy/2 , clean_ratelimit/2 , clean_quota/2 + , set_keepalive/2 ]). -export([ query/3 , format_channel_info/1 ]). --define(query_fun, {?MODULE, query}). --define(format_fun, {?MODULE, format_channel_info}). +-define(QUERY_FUN, {?MODULE, query}). +-define(FORMAT_FUN, {?MODULE, format_channel_info}). list(Bindings, Params) when map_size(Bindings) == 0 -> fence(fun() -> - emqx_mgmt_api:cluster_query(Params, ?CLIENT_QS_SCHEMA, ?query_fun) + emqx_mgmt_api:cluster_query(Params, ?CLIENT_QS_SCHEMA, ?QUERY_FUN) end); list(#{node := Node}, Params) when Node =:= node() -> fence(fun() -> - emqx_mgmt_api:node_query(Node, Params, ?CLIENT_QS_SCHEMA, ?query_fun) + emqx_mgmt_api:node_query(Node, Params, ?CLIENT_QS_SCHEMA, ?QUERY_FUN) end); list(Bindings = #{node := Node}, Params) -> @@ -169,16 +176,20 @@ fence(Func) -> end. lookup(#{node := Node, clientid := ClientId}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client(Node, {clientid, emqx_mgmt_util:urldecode(ClientId)}, ?format_fun)}); + minirest:return({ok, emqx_mgmt:lookup_client(Node, + {clientid, emqx_mgmt_util:urldecode(ClientId)}, ?FORMAT_FUN)}); lookup(#{clientid := ClientId}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client({clientid, emqx_mgmt_util:urldecode(ClientId)}, ?format_fun)}); + minirest:return({ok, emqx_mgmt:lookup_client( + {clientid, emqx_mgmt_util:urldecode(ClientId)}, ?FORMAT_FUN)}); lookup(#{node := Node, username := Username}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client(Node, {username, emqx_mgmt_util:urldecode(Username)}, ?format_fun)}); + minirest:return({ok, emqx_mgmt:lookup_client(Node, + {username, emqx_mgmt_util:urldecode(Username)}, ?FORMAT_FUN)}); lookup(#{username := Username}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client({username, emqx_mgmt_util:urldecode(Username)}, ?format_fun)}). + minirest:return({ok, emqx_mgmt:lookup_client({username, + emqx_mgmt_util:urldecode(Username)}, ?FORMAT_FUN)}). kickout(#{clientid := ClientId}, _Params) -> case emqx_mgmt:kickout_client(emqx_mgmt_util:urldecode(ClientId)) of @@ -204,7 +215,7 @@ list_acl_cache(#{clientid := ClientId}, _Params) -> set_ratelimit_policy(#{clientid := ClientId}, Params) -> P = [{conn_bytes_in, proplists:get_value(<<"conn_bytes_in">>, Params)}, {conn_messages_in, proplists:get_value(<<"conn_messages_in">>, Params)}], - case [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined] of + case filter_ratelimit_params(P) of [] -> minirest:return(); Policy -> case emqx_mgmt:set_ratelimit_policy(emqx_mgmt_util:urldecode(ClientId), Policy) of @@ -223,7 +234,7 @@ clean_ratelimit(#{clientid := ClientId}, _Params) -> set_quota_policy(#{clientid := ClientId}, Params) -> P = [{conn_messages_routing, proplists:get_value(<<"conn_messages_routing">>, Params)}], - case [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined] of + case filter_ratelimit_params(P) of [] -> minirest:return(); Policy -> case emqx_mgmt:set_quota_policy(emqx_mgmt_util:urldecode(ClientId), Policy) of @@ -233,6 +244,7 @@ set_quota_policy(#{clientid := ClientId}, Params) -> end end. + clean_quota(#{clientid := ClientId}, _Params) -> case emqx_mgmt:set_quota_policy(emqx_mgmt_util:urldecode(ClientId), []) of ok -> minirest:return(); @@ -240,6 +252,19 @@ clean_quota(#{clientid := ClientId}, _Params) -> {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) end. +set_keepalive(#{clientid := ClientId}, Params) -> + case proplists:get_value(<<"interval">>, Params) of + undefined -> + minirest:return({error, ?ERROR7, params_not_found}); + Interval0 -> + Interval = binary_to_integer(Interval0), + case emqx_mgmt:set_keepalive(emqx_mgmt_util:urldecode(ClientId), Interval) of + ok -> minirest:return(); + {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); + {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) + end + end. + %% @private %% S = 100,1s %% | 100KB, 1m @@ -266,7 +291,7 @@ format_channel_info({_Key, Info, Stats0}) -> ConnInfo = maps:get(conninfo, Info, #{}), Session = case maps:get(session, Info, #{}) of undefined -> #{}; - _Sess -> _Sess + Sess -> Sess end, SessCreated = maps:get(created_at, Session, maps:get(connected_at, ConnInfo)), Connected = case maps:get(conn_state, Info, connected) of @@ -306,7 +331,8 @@ format(Data) when is_map(Data)-> created_at => iolist_to_binary(strftime(CreatedAt div 1000))}, case maps:get(disconnected_at, Data, undefined) of undefined -> #{}; - DisconnectedAt -> #{disconnected_at => iolist_to_binary(strftime(DisconnectedAt div 1000))} + DisconnectedAt -> #{disconnected_at => + iolist_to_binary(strftime(DisconnectedAt div 1000))} end). format_acl_cache({{PubSub, Topic}, {AclResult, Timestamp}}) -> @@ -326,7 +352,8 @@ query({Qs, []}, Start, Limit) -> query({Qs, Fuzzy}, Start, Limit) -> Ms = qs2ms(Qs), MatchFun = match_fun(Ms, Fuzzy), - emqx_mgmt_api:traverse_table(emqx_channel_info, MatchFun, Start, Limit, fun format_channel_info/1). + emqx_mgmt_api:traverse_table(emqx_channel_info, MatchFun, + Start, Limit, fun format_channel_info/1). %%-------------------------------------------------------------------- %% Match funcs @@ -352,7 +379,7 @@ escape(B) when is_binary(B) -> run_fuzzy_match(_, []) -> true; -run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) -> +run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE} | Fuzzy]) -> Val = case maps:get(Key, ClientInfo, "") of undefined -> ""; V -> V @@ -406,6 +433,9 @@ ms(connected_at, X) -> ms(created_at, X) -> #{session => #{created_at => X}}. +filter_ratelimit_params(P) -> + [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined]. + %%-------------------------------------------------------------------- %% EUnits %%-------------------------------------------------------------------- diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index e45acfd42..f3bc42f3a 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -29,6 +29,8 @@ -define(HOST, "http://127.0.0.1:8081/"). +-elvis([{elvis_style, line_length, disable}]). + -define(API_VERSION, "v4"). -define(BASE_PATH, "api"). @@ -76,30 +78,40 @@ t_alarms(_) -> ?assert(is_existing(alarm2, emqx_alarm:get_alarms(activated))), {ok, Return1} = request_api(get, api_path(["alarms/activated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return1))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return1))))), + ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return1))))), + ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return1))))), emqx_alarm:deactivate(alarm1), {ok, Return2} = request_api(get, api_path(["alarms"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return2))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return2))))), + ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return2))))), + ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return2))))), {ok, Return3} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return3))))), - ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return3))))), + ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return3))))), + ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return3))))), emqx_alarm:deactivate(alarm2), {ok, Return4} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return4))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return4))))), + ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return4))))), + ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return4))))), {ok, _} = request_api(delete, api_path(["alarms/deactivated"]), auth_header_()), {ok, Return5} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assertNot(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return5))))), - ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return5))))). + ?assertNot(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return5))))), + ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return5))))). t_apps(_) -> AppId = <<"123456">>, @@ -153,7 +165,8 @@ t_banned(_) -> [Banned] = get(<<"data">>, Result), ?assertEqual(Who, maps:get(<<"who">>, Banned)), - {ok, _} = request_api(delete, api_path(["banned", "clientid", binary_to_list(Who)]), auth_header_()), + {ok, _} = request_api(delete, api_path(["banned", "clientid", binary_to_list(Who)]), + auth_header_()), {ok, Result2} = request_api(get, api_path(["banned"]), auth_header_()), ?assertEqual([], get(<<"data">>, Result2)). @@ -205,40 +218,50 @@ t_clients(_) -> meck:new(emqx_mgmt, [passthrough, no_history]), meck:expect(emqx_mgmt, kickout_client, 1, fun(_) -> {error, undefined} end), - {ok, MeckRet1} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), + {ok, MeckRet1} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), + auth_header_()), ?assertEqual(?ERROR1, get(<<"code">>, MeckRet1)), meck:expect(emqx_mgmt, clean_acl_cache, 1, fun(_) -> {error, undefined} end), - {ok, MeckRet2} = request_api(delete, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), + {ok, MeckRet2} = request_api(delete, + api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), ?assertEqual(?ERROR1, get(<<"code">>, MeckRet2)), meck:expect(emqx_mgmt, list_acl_cache, 1, fun(_) -> {error, undefined} end), - {ok, MeckRet3} = request_api(get, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), + {ok, MeckRet3} = request_api(get, + api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), ?assertEqual(?ERROR1, get(<<"code">>, MeckRet3)), meck:unload(emqx_mgmt), - {ok, Ok} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), + {ok, Ok} = request_api(delete, + api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), ?assertEqual(?SUCCESS, get(<<"code">>, Ok)), timer:sleep(300), - {ok, Ok1} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), + {ok, Ok1} = request_api(delete, + api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), ?assertEqual(?SUCCESS, get(<<"code">>, Ok1)), - {ok, Clients6} = request_api(get, api_path(["clients"]), "_limit=100&_page=1", auth_header_()), + {ok, Clients6} = request_api(get, + api_path(["clients"]), "_limit=100&_page=1", auth_header_()), ?assertEqual(1, maps:get(<<"count">>, get(<<"meta">>, Clients6))), - {ok, NotFound1} = request_api(get, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), + {ok, NotFound1} = request_api(get, + api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), ?assertEqual(?ERROR12, get(<<"code">>, NotFound1)), - {ok, NotFound2} = request_api(delete, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), + {ok, NotFound2} = request_api(delete, + api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), ?assertEqual(?ERROR12, get(<<"code">>, NotFound2)), - {ok, EmptyAclCache} = request_api(get, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), + {ok, EmptyAclCache} = request_api(get, + api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), ?assertEqual(0, length(get(<<"data">>, EmptyAclCache))), - {ok, Ok1} = request_api(delete, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), + {ok, Ok1} = request_api(delete, + api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), ?assertEqual(?SUCCESS, get(<<"code">>, Ok1)). receive_exit(0) -> @@ -257,7 +280,8 @@ receive_exit(Count) -> t_listeners(_) -> {ok, _} = request_api(get, api_path(["listeners"]), auth_header_()), - {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "listeners"]), auth_header_()), + {ok, _} = request_api(get, + api_path(["nodes", atom_to_list(node()), "listeners"]), auth_header_()), meck:new(emqx_mgmt, [passthrough, no_history]), meck:expect(emqx_mgmt, list_listeners, 0, fun() -> [{node(), {error, undefined}}] end), {ok, Return} = request_api(get, api_path(["listeners"]), auth_header_()), @@ -268,10 +292,12 @@ t_listeners(_) -> t_metrics(_) -> {ok, _} = request_api(get, api_path(["metrics"]), auth_header_()), - {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), + {ok, _} = request_api(get, + api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), meck:new(emqx_mgmt, [passthrough, no_history]), meck:expect(emqx_mgmt, get_metrics, 1, fun(_) -> {error, undefined} end), - {ok, "{\"message\":\"undefined\"}"} = request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), + {ok, "{\"message\":\"undefined\"}"} = + request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), meck:unload(emqx_mgmt). t_nodes(_) -> @@ -348,7 +374,8 @@ t_acl_cache(_) -> {ok, _} = emqtt:connect(C1), {ok, _, _} = emqtt:subscribe(C1, Topic, 2), %% get acl cache, should not be empty - {ok, Result} = request_api(get, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), + {ok, Result} = request_api(get, + api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), #{<<"code">> := 0, <<"data">> := Caches} = jiffy:decode(list_to_binary(Result), [return_maps]), ?assert(length(Caches) > 0), ?assertMatch(#{<<"access">> := <<"subscribe">>, @@ -356,11 +383,14 @@ t_acl_cache(_) -> <<"result">> := <<"allow">>, <<"updated_time">> := _}, hd(Caches)), %% clear acl cache - {ok, Result2} = request_api(delete, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), + {ok, Result2} = request_api(delete, + api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), ?assertMatch(#{<<"code">> := 0}, jiffy:decode(list_to_binary(Result2), [return_maps])), %% get acl cache again, after the acl cache is cleared - {ok, Result3} = request_api(get, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), - #{<<"code">> := 0, <<"data">> := Caches3} = jiffy:decode(list_to_binary(Result3), [return_maps]), + {ok, Result3} = request_api(get, + api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), + #{<<"code">> := 0, <<"data">> := Caches3} + = jiffy:decode(list_to_binary(Result3), [return_maps]), ?assertEqual(0, length(Caches3)), ok = emqtt:disconnect(C1). @@ -482,12 +512,15 @@ t_pubsub(_) -> Topic_list = [<<"mytopic1">>, <<"mytopic2">>], [ {ok, _, [2]} = emqtt:subscribe(C1, Topics, 2) || Topics <- Topic_list], - Body1 = [ #{<<"clientid">> => ClientId, <<"topic">> => Topics, <<"qos">> => 2} || Topics <- Topic_list], + Body1 = [ #{<<"clientid">> => ClientId, + <<"topic">> => Topics, <<"qos">> => 2} || Topics <- Topic_list], {ok, Data1} = request_api(post, api_path(["mqtt/subscribe_batch"]), [], auth_header_(), Body1), loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data1), [return_maps]))), %% tests publish_batch - Body2 = [ #{<<"clientid">> => ClientId, <<"topic">> => Topics, <<"qos">> => 2, <<"retain">> => <<"false">>, <<"payload">> => #{body => "hello world"}} || Topics <- Topic_list ], + Body2 = [ #{<<"clientid">> => ClientId, <<"topic">> => Topics, <<"qos">> => 2, + <<"retain">> => <<"false">>, <<"payload">> => #{body => "hello world"}} + || Topics <- Topic_list ], {ok, Data2} = request_api(post, api_path(["mqtt/publish_batch"]), [], auth_header_(), Body2), loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data2), [return_maps]))), [ ?assert(receive @@ -499,7 +532,8 @@ t_pubsub(_) -> %% tests unsubscribe_batch Body3 = [#{<<"clientid">> => ClientId, <<"topic">> => Topics} || Topics <- Topic_list], - {ok, Data3} = request_api(post, api_path(["mqtt/unsubscribe_batch"]), [], auth_header_(), Body3), + {ok, Data3} = request_api(post, + api_path(["mqtt/unsubscribe_batch"]), [], auth_header_(), Body3), loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data3), [return_maps]))), ok = emqtt:disconnect(C1), @@ -523,7 +557,8 @@ t_routes_and_subscriptions(_) -> ?assertEqual([], get(<<"data">>, NonRoute)), {ok, NonSubscription} = request_api(get, api_path(["subscriptions"]), auth_header_()), ?assertEqual([], get(<<"data">>, NonSubscription)), - {ok, NonSubscription1} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), + {ok, NonSubscription1} = request_api(get, + api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), ?assertEqual([], get(<<"data">>, NonSubscription1)), {ok, NonSubscription2} = request_api(get, api_path(["subscriptions", binary_to_list(ClientId)]), @@ -552,11 +587,14 @@ t_routes_and_subscriptions(_) -> ?assertEqual(Topic, maps:get(<<"topic">>, Subscription)), ?assertEqual(ClientId, maps:get(<<"clientid">>, Subscription)), - {ok, Result3} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), + {ok, Result3} = request_api(get, + api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), - {ok, Result4} = request_api(get, api_path(["subscriptions", binary_to_list(ClientId)]), auth_header_()), + {ok, Result4} = request_api(get, + api_path(["subscriptions", binary_to_list(ClientId)]), auth_header_()), [Subscription] = get(<<"data">>, Result4), - {ok, Result4} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions", binary_to_list(ClientId)]) + {ok, Result4} = request_api(get, + api_path(["nodes", atom_to_list(node()), "subscriptions", binary_to_list(ClientId)]) , auth_header_()), ok = emqtt:disconnect(C1). @@ -566,7 +604,8 @@ t_stats(_) -> {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "stats"]), auth_header_()), meck:new(emqx_mgmt, [passthrough, no_history]), meck:expect(emqx_mgmt, get_stats, 1, fun(_) -> {error, undefined} end), - {ok, Return} = request_api(get, api_path(["nodes", atom_to_list(node()), "stats"]), auth_header_()), + {ok, Return} = request_api(get, + api_path(["nodes", atom_to_list(node()), "stats"]), auth_header_()), ?assertEqual(<<"undefined">>, get(<<"message">>, Return)), meck:unload(emqx_mgmt). @@ -578,10 +617,15 @@ t_data(_) -> {ok, Data} = request_api(post, api_path(["data","export"]), [], auth_header_(), [#{}]), #{<<"filename">> := Filename, <<"node">> := Node} = emqx_ct_http:get_http_data(Data), {ok, DataList} = request_api(get, api_path(["data","export"]), auth_header_()), - ?assertEqual(true, lists:member(emqx_ct_http:get_http_data(Data), emqx_ct_http:get_http_data(DataList))), + ?assertEqual(true, + lists:member(emqx_ct_http:get_http_data(Data), emqx_ct_http:get_http_data(DataList))), - ?assertMatch({ok, _}, request_api(post, api_path(["data","import"]), [], auth_header_(), #{<<"filename">> => Filename, <<"node">> => Node})), - ?assertMatch({ok, _}, request_api(post, api_path(["data","import"]), [], auth_header_(), #{<<"filename">> => Filename})), + ?assertMatch({ok, _}, request_api(post, + api_path(["data","import"]), [], auth_header_(), + #{<<"filename">> => Filename, <<"node">> => Node})), + ?assertMatch({ok, _}, + request_api(post, api_path(["data","import"]), [], auth_header_(), + #{<<"filename">> => Filename})), application:stop(emqx_rule_engine), application:stop(emqx_dahboard), ok. @@ -596,9 +640,26 @@ t_data_import_content(_) -> Dir = emqx:get_env(data_dir), {ok, Bin} = file:read_file(filename:join(Dir, Filename)), Content = emqx_json:decode(Bin), - ?assertMatch({ok, "{\"code\":0}"}, request_api(post, api_path(["data","import"]), [], auth_header_(), Content)), + ?assertMatch({ok, "{\"code\":0}"}, + request_api(post, api_path(["data","import"]), [], auth_header_(), Content)), application:stop(emqx_rule_engine), - application:stop(emqx_dahboard). + application:stop(emqx_dashboard). + +t_keepalive(_Config) -> + application:ensure_all_started(emqx_dashboard), + Username = "user_keepalive", + ClientId = "client_keepalive", + AuthHeader = auth_header_(), + Path = api_path(["clients", ClientId, "keepalive"]), + {ok, NotFound} = request_api(put, Path, "interval=5", AuthHeader, [#{}]), + ?assertEqual("{\"message\":\"not_found\",\"code\":112}", NotFound), + {ok, C1} = emqtt:start_link(#{username => Username, clientid => ClientId}), + {ok, _} = emqtt:connect(C1), + {ok, Ok} = request_api(put, Path, "interval=5", AuthHeader, [#{}]), + ?assertEqual("{\"code\":0}", Ok), + emqtt:disconnect(C1), + application:stop(emqx_dashboard), + ok. request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index c3ffb9f90..a423fb9b7 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQ X Retainer"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 5f164d99e..4c49a1cb3 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -102,7 +102,7 @@ -type(reply() :: {outgoing, emqx_types:packet()} | {outgoing, [emqx_types:packet()]} - | {event, conn_state()|updated} + | {event, conn_state() | updated} | {close, Reason :: atom()}). -type(replies() :: emqx_types:packet() | reply() | [reply()]). @@ -131,7 +131,7 @@ info(Channel) -> maps:from_list(info(?INFO_KEYS, Channel)). --spec(info(list(atom())|atom(), channel()) -> term()). +-spec(info(list(atom()) | atom(), channel()) -> term()). info(Keys, Channel) when is_list(Keys) -> [{Key, info(Key, Channel)} || Key <- Keys]; info(conninfo, #channel{conninfo = ConnInfo}) -> @@ -619,7 +619,7 @@ ensure_quota(PubRes, Channel = #channel{quota = Limiter}) -> -compile({inline, [puback_reason_code/1]}). puback_reason_code([]) -> ?RC_NO_MATCHING_SUBSCRIBERS; -puback_reason_code([_|_]) -> ?RC_SUCCESS. +puback_reason_code([_ | _]) -> ?RC_SUCCESS. -compile({inline, [after_message_acked/3]}). after_message_acked(ClientInfo, Msg, PubAckProps) -> @@ -638,7 +638,7 @@ process_subscribe(TopicFilters, SubProps, Channel) -> process_subscribe([], _SubProps, Channel, Acc) -> {lists:reverse(Acc), Channel}; -process_subscribe([Topic = {TopicFilter, SubOpts}|More], SubProps, Channel, Acc) -> +process_subscribe([Topic = {TopicFilter, SubOpts} | More], SubProps, Channel, Acc) -> case check_sub_caps(TopicFilter, SubOpts, Channel) of ok -> {ReasonCode, NChannel} = do_subscribe(TopicFilter, @@ -676,9 +676,9 @@ process_unsubscribe(TopicFilters, UnSubProps, Channel) -> process_unsubscribe([], _UnSubProps, Channel, Acc) -> {lists:reverse(Acc), Channel}; -process_unsubscribe([{TopicFilter, SubOpts}|More], UnSubProps, Channel, Acc) -> +process_unsubscribe([{TopicFilter, SubOpts} | More], UnSubProps, Channel, Acc) -> {RC, NChannel} = do_unsubscribe(TopicFilter, SubOpts#{unsub_props => UnSubProps}, Channel), - process_unsubscribe(More, UnSubProps, NChannel, [RC|Acc]). + process_unsubscribe(More, UnSubProps, NChannel, [RC | Acc]). do_unsubscribe(TopicFilter, SubOpts, Channel = #channel{clientinfo = ClientInfo = #{mountpoint := MountPoint}, @@ -938,6 +938,17 @@ handle_call({quota, Policy}, Channel) -> Quota = emqx_limiter:init(Zone, Policy), reply(ok, Channel#channel{quota = Quota}); +handle_call({keepalive, Interval}, Channel = #channel{keepalive = KeepAlive, + conninfo = ConnInfo}) -> + ClientId = info(clientid, Channel), + NKeepalive = emqx_keepalive:set(interval, Interval * 1000, KeepAlive), + NConnInfo = maps:put(keepalive, Interval, ConnInfo), + NChannel = Channel#channel{keepalive = NKeepalive, conninfo = NConnInfo}, + SockInfo = maps:get(sockinfo, emqx_cm:get_chan_info(ClientId), #{}), + ChanInfo1 = info(NChannel), + emqx_cm:set_chan_info(ClientId, ChanInfo1#{sockinfo => SockInfo}), + reply(ok, reset_timer(alive_timer, NChannel)); + handle_call(Req, Channel) -> ?LOG(error, "Unexpected call: ~p", [Req]), reply(ignored, Channel). diff --git a/src/emqx_keepalive.erl b/src/emqx_keepalive.erl index 8fba00f50..216999acc 100644 --- a/src/emqx_keepalive.erl +++ b/src/emqx_keepalive.erl @@ -20,9 +20,11 @@ , info/1 , info/2 , check/2 + , set/3 ]). -export_type([keepalive/0]). +-elvis([{elvis_style, no_if_expression, disable}]). -record(keepalive, { interval :: pos_integer(), @@ -49,7 +51,7 @@ info(#keepalive{interval = Interval, repeat => Repeat }. --spec(info(interval|statval|repeat, keepalive()) +-spec(info(interval | statval | repeat, keepalive()) -> non_neg_integer()). info(interval, #keepalive{interval = Interval}) -> Interval; @@ -71,3 +73,7 @@ check(NewVal, KeepAlive = #keepalive{statval = OldVal, true -> {error, timeout} end. +%% @doc Update keepalive's interval +-spec(set(interval, non_neg_integer(), keepalive()) -> keepalive()). +set(interval, Interval, KeepAlive) -> + KeepAlive#keepalive{interval = Interval}. From ad4d3fc652330ac1bc041c6872715041f070732b Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 12 Nov 2021 21:26:44 +0800 Subject: [PATCH 024/104] chore(emqx_retainer): refresh the timestamp when dispatch retained message (#6155) 1. refresh the timestamp when dispatch retained message 2. fix some elvis style error --- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- apps/emqx_retainer/src/emqx_retainer.erl | 37 +++++++++++++++----- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index a423fb9b7..4ef423b78 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQ X Retainer"}, - {vsn, "4.4.0"}, % strict semver, bump manually! + {vsn, "4.4.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 340e6929d..fcc5652cc 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -78,7 +78,8 @@ dispatch(Pid, Topic) -> false -> read_messages(Topic); true -> match_messages(Topic) end, - [Pid ! {deliver, Topic, Msg} || Msg <- sort_retained(Msgs)]. + Now = erlang:system_time(millisecond), + [Pid ! {deliver, Topic, refresh_timestamp_expiry(Msg, Now)} || Msg <- sort_retained(Msgs)]. %% RETAIN flag set to 1 and payload containing zero bytes on_message_publish(Msg = #message{flags = #{retain := true}, @@ -146,7 +147,7 @@ init([Env]) -> ok end, StatsFun = emqx_stats:statsfun('retained.count', 'retained.max'), - {ok, StatsTimer} = timer:send_interval(timer:seconds(1), stats), + StatsTimer = erlang:send_after(timer:seconds(1), self(), stats), State = #state{stats_fun = StatsFun, stats_timer = StatsTimer}, {ok, start_expire_timer(proplists:get_value(expiry_interval, Env, 0), State)}. @@ -155,7 +156,7 @@ start_expire_timer(0, State) -> start_expire_timer(undefined, State) -> State; start_expire_timer(Ms, State) -> - {ok, Timer} = timer:send_interval(Ms, expire), + Timer = erlang:send_after(Ms, self(), stats), State#state{expiry_timer = Timer}. handle_call(Req, _From, State) -> @@ -194,7 +195,7 @@ sort_retained([]) -> []; sort_retained([Msg]) -> [Msg]; sort_retained(Msgs) -> lists:sort(fun(#message{timestamp = Ts1}, #message{timestamp = Ts2}) -> - Ts1 =< Ts2 + Ts1 =< Ts2 end, Msgs). store_retained(Msg = #message{topic = Topic, payload = Payload}, Env) -> @@ -209,11 +210,13 @@ store_retained(Msg = #message{topic = Topic, payload = Payload}, Env) -> fun() -> case mnesia:read(?TAB, Topic) of [_] -> - mnesia:write(?TAB, #retained{topic = topic2tokens(Topic), - msg = Msg, - expiry_time = get_expiry_time(Msg, Env)}, write); + mnesia:write(?TAB, + #retained{topic = topic2tokens(Topic), + msg = Msg, + expiry_time = get_expiry_time(Msg, Env)}, write); [] -> - ?LOG(error, "Cannot retain message(topic=~s) for table is full!", [Topic]) + ?LOG(error, + "Cannot retain message(topic=~s) for table is full!", [Topic]) end end), ok; @@ -234,7 +237,8 @@ is_too_big(Size, Env) -> get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := 0}}}, _Env) -> 0; -get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := Interval}}, timestamp = Ts}, _Env) -> +get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := Interval}}, + timestamp = Ts}, _Env) -> Ts + Interval * 1000; get_expiry_time(#message{timestamp = Ts}, Env) -> case proplists:get_value(expiry_interval, Env, 0) of @@ -303,3 +307,18 @@ condition(Ws) -> false -> Ws1; _ -> (Ws1 -- ['#']) ++ '_' end. + +-spec(refresh_timestamp_expiry(emqx_types:message(), pos_integer()) -> emqx_types:message()). +refresh_timestamp_expiry(Msg = #message{headers = + #{properties := + #{'Message-Expiry-Interval' := Interval} = Props}, + timestamp = CreatedAt}, + Now) -> + Elapsed = max(0, Now - CreatedAt), + Interval1 = max(1, Interval - (Elapsed div 1000)), + emqx_message:set_header(properties, + Props#{'Message-Expiry-Interval' => Interval1}, + Msg#message{timestamp = Now}); + +refresh_timestamp_expiry(Msg, Now) -> + Msg#message{timestamp = Now}. From 6fb3ff1f9f183c444c9126dc06e025d4d8fe9fd2 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 5 Nov 2021 18:32:43 +0800 Subject: [PATCH 025/104] feat(exhook): expose headers for on_messages_publish hook --- apps/emqx_exhook/priv/protos/exhook.proto | 25 ++++++++++ apps/emqx_exhook/src/emqx_exhook_handler.erl | 48 +++++++++++++++++-- .../emqx_exhook/test/emqx_exhook_demo_svr.erl | 11 +++-- src/emqx_types.erl | 1 + 4 files changed, 78 insertions(+), 7 deletions(-) diff --git a/apps/emqx_exhook/priv/protos/exhook.proto b/apps/emqx_exhook/priv/protos/exhook.proto index 72ba26581..639066c6a 100644 --- a/apps/emqx_exhook/priv/protos/exhook.proto +++ b/apps/emqx_exhook/priv/protos/exhook.proto @@ -358,6 +358,31 @@ message Message { bytes payload = 6; uint64 timestamp = 7; + + // The key of header can be: + // - username: + // * Readonly + // * The username of sender client + // * Value type: utf8 string + // - protocol: + // * Readonly + // * The protocol name of sender client + // * Value type: string enum with "mqtt", "mqtt-sn", ... + // - peerhost: + // * Readonly + // * The peerhost of sender client + // * Value type: ip address string + // - allow_publish: + // * Writable + // * Whether to allow the message to be published by emqx + // * Value type: string enum with "true", "false", default is "true" + // + // Notes: All header may be missing, which means that the message does not + // carry these headers. We can guarantee that clients coming from MQTT, + // MQTT-SN, CoAP, LwM2M and other natively supported protocol clients will + // carry these headers, but there is no guarantee that messages published + // by other means will do, e.g. messages published by HTTP-API + map headers = 8; } message Property { diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index f3964dc42..efdf7f8b9 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -258,17 +258,57 @@ clientinfo(ClientInfo = cn => maybe(maps:get(cn, ClientInfo, undefined)), dn => maybe(maps:get(dn, ClientInfo, undefined))}. -message(#message{id = Id, qos = Qos, from = From, topic = Topic, payload = Payload, timestamp = Ts}) -> +message(#message{id = Id, qos = Qos, from = From, topic = Topic, + payload = Payload, timestamp = Ts, headers = Headers}) -> #{node => stringfy(node()), id => emqx_guid:to_hexstr(Id), qos => Qos, from => stringfy(From), topic => Topic, payload => Payload, - timestamp => Ts}. + timestamp => Ts, + headers => headers(Headers) + }. -assign_to_message(#{qos := Qos, topic := Topic, payload := Payload}, Message) -> - Message#message{qos = Qos, topic = Topic, payload = Payload}. +headers(undefined) -> + #{}; +headers(Headers) -> + Ls = [username, protocol, peerhost, allow_publish], + maps:fold( + fun + (_, undefined, Acc) -> + Acc; %% Ignore undefined value + (K, V, Acc) -> + case lists:member(K, Ls) of + true -> + Acc#{atom_to_binary(K) => bin(K, V)}; + _ -> + Acc + end + end, #{}, Headers). + +bin(K, V) when K == username; + K == protocol; + K == allow_publish -> + bin(V); +bin(peerhost, V) -> + bin(inet:ntoa(V)). + +bin(V) when is_binary(V) -> V; +bin(V) when is_atom(V) -> atom_to_binary(V); +bin(V) when is_list(V) -> iolist_to_binary(V). + +assign_to_message(InMessage = #{qos := Qos, topic := Topic, + payload := Payload}, Message) -> + NMsg = Message#message{qos = Qos, topic = Topic, payload = Payload}, + enrich_header(maps:get(headers, InMessage, #{}), NMsg). + +enrich_header(Headers, Message) -> + AllowPub = case maps:get(<<"allow_publish">>, Headers, <<"true">>) of + <<"false">> -> false; + _ -> true + end, + emqx_message:set_header(allow_publish, AllowPub, Message). topicfilters(Tfs) when is_list(Tfs) -> [#{name => Topic, qos => Qos} || {Topic, #{qos := Qos}} <- Tfs]. diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl index c2db04dd4..fd8a5f9a3 100644 --- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl +++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl @@ -295,14 +295,14 @@ on_session_terminated(Req, Md) -> | {error, grpc_cowboy_h:error_response()}. on_message_publish(#{message := #{from := From} = Msg} = Req, Md) -> ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), + io:format(standard_error, "fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), %% some cases for testing case From of <<"baduser">> -> - NMsg = Msg#{qos => 0, + NMsg = deny(Msg#{qos => 0, topic => <<"">>, payload => <<"">> - }, + }), {ok, #{type => 'STOP_AND_RETURN', value => {message, NMsg}}, Md}; <<"gooduser">> -> @@ -314,6 +314,11 @@ on_message_publish(#{message := #{from := From} = Msg} = Req, Md) -> {ok, #{type => 'IGNORE'}, Md} end. +deny(Msg) -> + NHeader = maps:put(<<"allow_publish">>, <<"false">>, + maps:get(headers, Msg, #{})), + maps:put(headers, NHeader, Msg). + -spec on_message_delivered(emqx_exhook_pb:message_delivered_request(), grpc:metadata()) -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} | {error, grpc_cowboy_h:error_response()}. diff --git a/src/emqx_types.erl b/src/emqx_types.erl index 3e53eafd1..410ed5a27 100644 --- a/src/emqx_types.erl +++ b/src/emqx_types.erl @@ -193,6 +193,7 @@ username => username(), peerhost => peerhost(), properties => properties(), + allow_publish => boolean(), atom() => term()}). -type(banned() :: #banned{}). From ef2a5c1dc71722d35aafc307d516a99503c7e5cd Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 8 Nov 2021 11:11:16 +0800 Subject: [PATCH 026/104] chore(exhook): fix diaylzer warnings --- apps/emqx_exhook/src/emqx_exhook_handler.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index efdf7f8b9..45e6bf082 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -270,8 +270,6 @@ message(#message{id = Id, qos = Qos, from = From, topic = Topic, headers => headers(Headers) }. -headers(undefined) -> - #{}; headers(Headers) -> Ls = [username, protocol, peerhost, allow_publish], maps:fold( From 012c7415926d2cf11cc1e52917c4774dbb89021e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 8 Nov 2021 11:11:37 +0800 Subject: [PATCH 027/104] chore(exhook): update appup.src --- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook.appup.src | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 46223d212..b386bcaca 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,6 +1,6 @@ {application, emqx_exhook, [{description, "EMQ X Extension for Hook"}, - {vsn, "4.3.4"}, + {vsn, "4.3.5"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index d6a699c33..e78afef1e 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -1,15 +1,17 @@ %% -*-: erlang -*- {VSN, - [ - {<<"4.3.[0-3]">>, [ - {restart_application, emqx_exhook} - ]}, - {<<".*">>, []} + [{"4.3.4", + [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}]}, + {<<"4\\.3\\.[0-3]+">>, + [{restart_application,emqx_exhook}]}, + {<<".*">>, []} ], - [ - {<<"4.3.[0-3]">>, [ - {restart_application, emqx_exhook} - ]}, - {<<".*">>, []} + [{"4.3.4", + [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}]}, + {<<"4\\.3\\.[0-3]+">>, + [{restart_application,emqx_exhook}]}, + {<<".*">>, []} ] }. From b756e7d17a4e73213329d518cd5e0217355cf268 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 8 Nov 2021 14:15:32 +0800 Subject: [PATCH 028/104] chore: upgrade grpc to 0.6.4 --- apps/emqx_exhook/rebar.config | 2 +- apps/emqx_exproto/rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config index 3529b6314..d1cc4d778 100644 --- a/apps/emqx_exhook/rebar.config +++ b/apps/emqx_exhook/rebar.config @@ -5,7 +5,7 @@ ]}. {deps, - [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.3"}}} + [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}} ]}. {grpc, diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_exproto/rebar.config index 4ad1aa192..da868de82 100644 --- a/apps/emqx_exproto/rebar.config +++ b/apps/emqx_exproto/rebar.config @@ -13,7 +13,7 @@ ]}. {deps, - [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.3"}}} + [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}} ]}. {grpc, From c170d076e32ad72db408a85a41644db2c8df94e2 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 8 Nov 2021 14:16:26 +0800 Subject: [PATCH 029/104] feat(exhook): expose process pool_size for grpc client --- apps/emqx_exhook/etc/emqx_exhook.conf | 5 +++++ apps/emqx_exhook/priv/emqx_exhook.schema | 4 ++++ apps/emqx_exhook/src/emqx_exhook_mngr.erl | 11 +++++++++++ apps/emqx_exhook/src/emqx_exhook_server.erl | 3 ++- apps/emqx_exhook/src/emqx_exhook_sup.erl | 5 +++-- 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/apps/emqx_exhook/etc/emqx_exhook.conf b/apps/emqx_exhook/etc/emqx_exhook.conf index ffb71e43b..23895f902 100644 --- a/apps/emqx_exhook/etc/emqx_exhook.conf +++ b/apps/emqx_exhook/etc/emqx_exhook.conf @@ -24,6 +24,11 @@ ## Value: false | Duration #exhook.auto_reconnect = 60s +## The process pool size for gRPC client +## +## Default: Equals cpu cores +## Value: Integer +#exhook.pool_size = 16 ##-------------------------------------------------------------------- ## The Hook callback servers diff --git a/apps/emqx_exhook/priv/emqx_exhook.schema b/apps/emqx_exhook/priv/emqx_exhook.schema index d11001c0d..f55913d72 100644 --- a/apps/emqx_exhook/priv/emqx_exhook.schema +++ b/apps/emqx_exhook/priv/emqx_exhook.schema @@ -26,6 +26,10 @@ end end}. +{mapping, "exhook.pool_size", "emqx_exhook.pool_size", [ + {datatype, integer} +]}. + {mapping, "exhook.server.$name.url", "emqx_exhook.servers", [ {datatype, string} ]}. diff --git a/apps/emqx_exhook/src/emqx_exhook_mngr.erl b/apps/emqx_exhook/src/emqx_exhook_mngr.erl index cadd5eb37..54e106f13 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mngr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mngr.erl @@ -36,6 +36,8 @@ , server/1 , put_request_failed_action/1 , get_request_failed_action/0 + , put_pool_size/1 + , get_pool_size/0 ]). %% gen_server callbacks @@ -117,6 +119,9 @@ init([Servers, AutoReconnect, ReqOpts0]) -> put_request_failed_action( maps:get(request_failed_action, ReqOpts0, deny) ), + put_pool_size( + maps:get(pool_size, ReqOpts0, erlang:system_info(schedulers)) + ), %% Load the hook servers ReqOpts = maps:without([request_failed_action], ReqOpts0), @@ -286,6 +291,12 @@ put_request_failed_action(Val) -> get_request_failed_action() -> persistent_term:get({?APP, request_failed_action}). +put_pool_size(Val) -> + persistent_term:put({?APP, pool_size}, Val). + +get_pool_size() -> + persistent_term:get({?APP, pool_size}). + save(Name, ServerState) -> Saved = persistent_term:get(?APP, []), persistent_term:put(?APP, lists:reverse([Name | Saved])), diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 7df5b643c..123cbb558 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -131,7 +131,8 @@ channel_opts(Opts) -> transport_opts => SslOpts}}; _ -> #{} end, - {SvrAddr, ClientOpts}. + NClientOpts = ClientOpts#{pool_size => emqx_exhook_mngr:get_pool_size()}, + {SvrAddr, NClientOpts}. format_http_uri(Scheme, Host0, Port) -> Host = case is_tuple(Host0) of diff --git a/apps/emqx_exhook/src/emqx_exhook_sup.erl b/apps/emqx_exhook/src/emqx_exhook_sup.erl index e9c405de0..c92fd6ca4 100644 --- a/apps/emqx_exhook/src/emqx_exhook_sup.erl +++ b/apps/emqx_exhook/src/emqx_exhook_sup.erl @@ -54,7 +54,8 @@ auto_reconnect() -> request_options() -> #{timeout => env(request_timeout, 5000), - request_failed_action => env(request_failed_action, deny) + request_failed_action => env(request_failed_action, deny), + pool_size => env(pool_size, erlang:system_info(schedulers)) }. env(Key, Def) -> @@ -67,7 +68,7 @@ env(Key, Def) -> -spec start_grpc_client_channel( string(), uri_string:uri_string(), - grpc_client:options()) -> {ok, pid()} | {error, term()}. + grpc_client_sup:options()) -> {ok, pid()} | {error, term()}. start_grpc_client_channel(Name, SvrAddr, Options) -> grpc_client_sup:create_channel_pool(Name, SvrAddr, Options). From 44008b9a6da0aa39354be5c7a96197297942023e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 8 Nov 2021 15:05:29 +0800 Subject: [PATCH 030/104] chore: fix compiling warnings --- apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index 0947bdaca..794eedd6b 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -171,7 +171,7 @@ t_subscribe_case02(_) -> {ok, Socket} = gen_udp:open(0, [binary]), ClientId = ?CLIENTID, - send_connect_msg(Socket, ?CLIENTID), + send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), Topic1 = ?PREDEF_TOPIC_NAME1, From 5922521e499b8bb2b551e5ca2b8564daefd49310 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 8 Nov 2021 15:05:54 +0800 Subject: [PATCH 031/104] test(props): cover messages headers --- apps/emqx_exhook/src/emqx_exhook_handler.erl | 14 +++++++++----- .../emqx_exhook/test/emqx_exhook_demo_svr.erl | 9 +++++++-- .../test/props/prop_exhook_hooks.erl | 19 +++++++++++++------ 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index 45e6bf082..ea6cdf5bc 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -50,6 +50,7 @@ %% Utils -export([ message/1 + , headers/1 , stringfy/1 , merge_responsed_bool/2 , merge_responsed_message/2 @@ -302,11 +303,14 @@ assign_to_message(InMessage = #{qos := Qos, topic := Topic, enrich_header(maps:get(headers, InMessage, #{}), NMsg). enrich_header(Headers, Message) -> - AllowPub = case maps:get(<<"allow_publish">>, Headers, <<"true">>) of - <<"false">> -> false; - _ -> true - end, - emqx_message:set_header(allow_publish, AllowPub, Message). + case maps:get(<<"allow_publish">>, Headers, undefined) of + <<"false">> -> + emqx_message:set_header(allow_publish, false, Message); + <<"true">> -> + emqx_message:set_header(allow_publish, true, Message); + _ -> + Message + end. topicfilters(Tfs) when is_list(Tfs) -> [#{name => Topic, qos => Qos} || {Topic, #{qos := Qos}} <- Tfs]. diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl index fd8a5f9a3..6e9691c3e 100644 --- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl +++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl @@ -306,8 +306,8 @@ on_message_publish(#{message := #{from := From} = Msg} = Req, Md) -> {ok, #{type => 'STOP_AND_RETURN', value => {message, NMsg}}, Md}; <<"gooduser">> -> - NMsg = Msg#{topic => From, - payload => From}, + NMsg = allow(Msg#{topic => From, + payload => From}), {ok, #{type => 'STOP_AND_RETURN', value => {message, NMsg}}, Md}; _ -> @@ -319,6 +319,11 @@ deny(Msg) -> maps:get(headers, Msg, #{})), maps:put(headers, NHeader, Msg). +allow(Msg) -> + NHeader = maps:put(<<"allow_publish">>, <<"true">>, + maps:get(headers, Msg, #{})), + maps:put(headers, NHeader, Msg). + -spec on_message_delivered(emqx_exhook_pb:message_delivered_request(), grpc:metadata()) -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} | {error, grpc_cowboy_h:error_response()}. diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl index 24f45c8b0..88eba8f11 100644 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl @@ -299,19 +299,24 @@ prop_message_publish() -> _ -> ExpectedOutMsg = case emqx_message:from(Msg) of <<"baduser">> -> - MsgMap = emqx_message:to_map(Msg), + MsgMap = #{headers := Headers} + = emqx_message:to_map(Msg), emqx_message:from_map( MsgMap#{qos => 0, topic => <<"">>, - payload => <<"">> + payload => <<"">>, + headers => maps:put(allow_publish, false, Headers) }); <<"gooduser">> = From -> - MsgMap = emqx_message:to_map(Msg), + MsgMap = #{headers := Headers} + = emqx_message:to_map(Msg), emqx_message:from_map( MsgMap#{topic => From, - payload => From + payload => From, + headers => maps:put(allow_publish, true, Headers) }); - _ -> Msg + _ -> + Msg end, ?assertEqual(ExpectedOutMsg, OutMsg), @@ -464,7 +469,9 @@ from_message(Msg) -> from => stringfy(emqx_message:from(Msg)), topic => emqx_message:topic(Msg), payload => emqx_message:payload(Msg), - timestamp => emqx_message:timestamp(Msg) + timestamp => emqx_message:timestamp(Msg), + headers => emqx_exhook_handler:headers( + emqx_message:get_headers(Msg)) }. %%-------------------------------------------------------------------- From 641f36514ff6292705e31e89de2c503f6a24db22 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 9 Nov 2021 11:52:47 +0800 Subject: [PATCH 032/104] chore: fix elvis warnings --- apps/emqx_exhook/src/emqx_exhook_handler.erl | 17 +- apps/emqx_exhook/src/emqx_exhook_mngr.erl | 6 +- apps/emqx_exhook/src/emqx_exhook_server.erl | 36 +- apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl | 375 ++++++++++++++----- src/emqx_types.erl | 2 +- 5 files changed, 307 insertions(+), 129 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index ea6cdf5bc..63d41d8eb 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -63,6 +63,8 @@ , call_fold/3 ]). +-elvis([{elvis_style, god_modules, disable}]). + %%-------------------------------------------------------------------- %% Clients %%-------------------------------------------------------------------- @@ -341,11 +343,7 @@ merge_responsed_bool(_Req, #{type := 'IGNORE'}) -> ignore; merge_responsed_bool(Req, #{type := Type, value := {bool_result, NewBool}}) when is_boolean(NewBool) -> - NReq = Req#{result => NewBool}, - case Type of - 'CONTINUE' -> {ok, NReq}; - 'STOP_AND_RETURN' -> {stop, NReq} - end; + {ret(Type), Req#{result => NewBool}}; merge_responsed_bool(_Req, Resp) -> ?LOG(warning, "Unknown responsed value ~0p to merge to callback chain", [Resp]), ignore. @@ -353,11 +351,10 @@ merge_responsed_bool(_Req, Resp) -> merge_responsed_message(_Req, #{type := 'IGNORE'}) -> ignore; merge_responsed_message(Req, #{type := Type, value := {message, NMessage}}) -> - NReq = Req#{message => NMessage}, - case Type of - 'CONTINUE' -> {ok, NReq}; - 'STOP_AND_RETURN' -> {stop, NReq} - end; + {ret(Type), Req#{message => NMessage}}; merge_responsed_message(_Req, Resp) -> ?LOG(warning, "Unknown responsed value ~0p to merge to callback chain", [Resp]), ignore. + +ret('CONTINUE') -> ok; +ret('STOP_AND_RETURN') -> stop. diff --git a/apps/emqx_exhook/src/emqx_exhook_mngr.erl b/apps/emqx_exhook/src/emqx_exhook_mngr.erl index 54e106f13..0f3c8b8ab 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mngr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mngr.erl @@ -86,11 +86,11 @@ start_link(Servers, AutoReconnect, ReqOpts) -> gen_server:start_link(?MODULE, [Servers, AutoReconnect, ReqOpts], []). --spec enable(pid(), atom()|string()) -> ok | {error, term()}. +-spec enable(pid(), atom() | string()) -> ok | {error, term()}. enable(Pid, Name) -> call(Pid, {load, Name}). --spec disable(pid(), atom()|string()) -> ok | {error, term()}. +-spec disable(pid(), atom() | string()) -> ok | {error, term()}. disable(Pid, Name) -> call(Pid, {unload, Name}). @@ -141,7 +141,7 @@ load_all_servers(Servers, ReqOpts) -> load_all_servers(Servers, ReqOpts, #{}, #{}). load_all_servers([], _Request, Waiting, Running) -> {Waiting, Running}; -load_all_servers([{Name, Options}|More], ReqOpts, Waiting, Running) -> +load_all_servers([{Name, Options} | More], ReqOpts, Waiting, Running) -> {NWaiting, NRunning} = case emqx_exhook_server:load(Name, Options, ReqOpts) of {ok, ServerState} -> diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 123cbb558..9e88d774d 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -77,6 +77,8 @@ -dialyzer({nowarn_function, [inc_metrics/2]}). +-elvis([{elvis_style, dont_repeat_yourself, disable}]). + %%-------------------------------------------------------------------- %% Load/Unload APIs %%-------------------------------------------------------------------- @@ -125,7 +127,11 @@ channel_opts(Opts) -> SvrAddr = format_http_uri(Scheme, Host, Port), ClientOpts = case Scheme of https -> - SslOpts = lists:keydelete(ssl, 1, proplists:get_value(ssl_options, Opts, [])), + SslOpts = lists:keydelete( + ssl, + 1, + proplists:get_value(ssl_options, Opts, []) + ), #{gun_opts => #{transport => ssl, transport_opts => SslOpts}}; @@ -175,16 +181,18 @@ resovle_hookspec(HookSpecs) when is_list(HookSpecs) -> case maps:get(name, HookSpec, undefined) of undefined -> Acc; Name0 -> - Name = try binary_to_existing_atom(Name0, utf8) catch T:R:_ -> {T,R} end, - case lists:member(Name, AvailableHooks) of - true -> - case lists:member(Name, MessageHooks) of - true -> - Acc#{Name => #{topics => maps:get(topics, HookSpec, [])}}; - _ -> - Acc#{Name => #{}} - end; - _ -> error({unknown_hookpoint, Name}) + Name = try + binary_to_existing_atom(Name0, utf8) + catch T:R:_ -> {T,R} + end, + case {lists:member(Name, AvailableHooks), + lists:member(Name, MessageHooks)} of + {false, _} -> + error({unknown_hookpoint, Name}); + {true, false} -> + Acc#{Name => #{}}; + {true, true} -> + Acc#{Name => #{topics => maps:get(topics, HookSpec, [])}} end end end, #{}, HookSpecs). @@ -256,7 +264,7 @@ call(Hookpoint, Req, #server{name = ChannName, options = ReqOpts, %% @private inc_metrics(IncFun, Name) when is_function(IncFun) -> %% BACKW: e4.2.0-e4.2.2 - {env, [Prefix|_]} = erlang:fun_info(IncFun, env), + {env, [Prefix | _]} = erlang:fun_info(IncFun, env), inc_metrics(Prefix, Name); inc_metrics(Prefix, Name) when is_list(Prefix) -> emqx_metrics:inc(list_to_atom(Prefix ++ atom_to_list(Name))). @@ -272,8 +280,8 @@ do_call(ChannName, Fun, Req, ReqOpts) -> Options = ReqOpts#{channel => ChannName}, ?LOG(debug, "Call ~0p:~0p(~0p, ~0p)", [?PB_CLIENT_MOD, Fun, Req, Options]), case catch apply(?PB_CLIENT_MOD, Fun, [Req, Options]) of - {ok, Resp, _Metadata} -> - ?LOG(debug, "Response {ok, ~0p, ~0p}", [Resp, _Metadata]), + {ok, Resp, Metadata} -> + ?LOG(debug, "Response {ok, ~0p, ~0p}", [Resp, Metadata]), {ok, Resp}; {error, {Code, Msg}, _Metadata} -> ?LOG(error, "CALL ~0p:~0p(~0p, ~0p) response errcode: ~0p, errmsg: ~0p", diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index 794eedd6b..27215cd4f 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -50,6 +50,9 @@ %% erlang:system_time should be unique and random enough -define(CLIENTID, iolist_to_binary([atom_to_list(?FUNCTION_NAME), "-", integer_to_list(erlang:system_time())])). + +-elvis([{elvis_style, dont_repeat_yourself, disable}]). + %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- @@ -66,8 +69,10 @@ end_per_suite(_) -> emqx_ct_helpers:stop_apps([emqx_sn]). set_special_confs(emqx) -> - application:set_env(emqx, plugins_loaded_file, - emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")); + application:set_env( + emqx, + plugins_loaded_file, + emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")); set_special_confs(emqx_sn) -> application:set_env(emqx_sn, enable_qos3, ?ENABLE_QOS3), application:set_env(emqx_sn, enable_stats, true), @@ -113,7 +118,8 @@ t_subscribe(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), TopicName1 = <<"abcD">>, send_register_msg(Socket, TopicName1, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, + receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId), ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, @@ -145,7 +151,8 @@ t_subscribe_case01(_) -> TopicName1 = <<"abcD">>, send_register_msg(Socket, TopicName1, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, + receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId), ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ReturnCode>>, @@ -166,7 +173,7 @@ t_subscribe_case02(_) -> Will = 0, CleanSession = 0, MsgId = 1, - TopicId = ?PREDEF_TOPIC_ID1, %this TopicId is the predefined topic id corresponding to ?PREDEF_TOPIC_NAME1 + TopicId = ?PREDEF_TOPIC_ID1, ReturnCode = 0, {ok, Socket} = gen_udp:open(0, [binary]), @@ -176,7 +183,8 @@ t_subscribe_case02(_) -> Topic1 = ?PREDEF_TOPIC_NAME1, send_register_msg(Socket, Topic1, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, + receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, TopicId, MsgId), ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ReturnCode>>, @@ -206,9 +214,11 @@ t_subscribe_case03(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_short_topic(Socket, QoS, <<"te">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, + CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_unsubscribe_msg_short_topic(Socket, <<"te">>, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -217,8 +227,12 @@ t_subscribe_case03(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). -%%In this case We use predefined topic name to register and subcribe, and expect to receive the corresponding predefined topic id but not a new generated topic id from broker. We design this case to illustrate -%% emqx_sn_gateway's compatibility of dealing with predefined and normal topics. Once we give more restrictions to different topic id type, this case would be deleted or modified. +%% In this case We use predefined topic name to register and subcribe, and +%% expect to receive the corresponding predefined topic id but not a new +%% generated topic id from broker. We design this case to illustrate +%% emqx_sn_gateway's compatibility of dealing with predefined and normal topics. +%% Once we give more restrictions to different topic id type, this case would +%% be deleted or modified. t_subscribe_case04(_) -> Dup = 0, QoS = 0, @@ -226,7 +240,7 @@ t_subscribe_case04(_) -> Will = 0, CleanSession = 0, MsgId = 1, - TopicId = ?PREDEF_TOPIC_ID1, %this TopicId is the predefined topic id corresponding to ?PREDEF_TOPIC_NAME1 + TopicId = ?PREDEF_TOPIC_ID1, ReturnCode = 0, {ok, Socket} = gen_udp:open(0, [binary]), ClientId = ?CLIENTID, @@ -234,10 +248,14 @@ t_subscribe_case04(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), Topic1 = ?PREDEF_TOPIC_NAME1, send_register_msg(Socket, Topic1, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, + receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, Topic1, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_unsubscribe_msg_normal_topic(Socket, Topic1, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -264,19 +282,30 @@ t_subscribe_case05(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_register_msg(Socket, <<"abcD">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"abcD">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"/sport/#">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"/a/+/water">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"/Tom/Home">>, MsgId), ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, @@ -306,19 +335,32 @@ t_subscribe_case06(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_register_msg(Socket, <<"abc">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, + receive_response(Socket) + ), send_register_msg(Socket, <<"/blue/#">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, + receive_response(Socket) + ), send_register_msg(Socket, <<"/blue/+/white">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, + receive_response(Socket) + ), send_register_msg(Socket, <<"/$sys/rain">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId2:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId2:16, MsgId:16, 0:8>>, + receive_response(Socket) + ), send_subscribe_msg_short_topic(Socket, QoS, <<"Q2">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_unsubscribe_msg_normal_topic(Socket, <<"Q2">>, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -342,8 +384,11 @@ t_subscribe_case07(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, TopicId1, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, + receive_response(Socket) + ), send_unsubscribe_msg_predefined_topic(Socket, TopicId2, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -365,8 +410,11 @@ t_subscribe_case08(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_reserved_topic(Socket, QoS, TopicId2, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -390,15 +438,20 @@ t_publish_negqos_case09(_) -> send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_normal_topic(Socket, NegQoS, MsgId1, TopicId1, Payload1), timer:sleep(100), case ?ENABLE_QOS3 of true -> - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What) end, @@ -431,7 +484,9 @@ t_publish_qos0_case01(_) -> send_publish_msg_normal_topic(Socket, QoS, MsgId1, TopicId1, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -453,15 +508,20 @@ t_publish_qos0_case02(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId1, PredefTopicId, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2, PredefTopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2, + PredefTopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -484,15 +544,20 @@ t_publish_qos0_case3(_) -> Topic = <<"/a/b/c">>, send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId1, TopicId, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -514,8 +579,11 @@ t_publish_qos0_case04(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, <<"#">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 2, Payload1 = <<20, 21, 22, 23>>, @@ -523,7 +591,9 @@ t_publish_qos0_case04(_) -> send_publish_msg_short_topic(Socket, QoS, MsgId1, Topic, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2, Topic/binary, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2, + Topic/binary, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -544,8 +614,11 @@ t_publish_qos0_case05(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_short_topic(Socket, QoS, <<"/#">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -567,15 +640,20 @@ t_publish_qos0_case06(_) -> Topic = <<"abc">>, send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_normal_topic(Socket, QoS, MsgId1, TopicId1, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -597,16 +675,25 @@ t_publish_qos1_case01(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Payload1 = <<20, 21, 22, 23>>, send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1), - ?assertEqual(<<7, ?SN_PUBACK, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicId1:16, + MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), timer:sleep(100), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), gen_udp:close(Socket). @@ -625,12 +712,18 @@ t_publish_qos1_case02(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId, PredefTopicId, Payload1), - ?assertEqual(<<7, ?SN_PUBACK, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, PredefTopicId:16, + MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), timer:sleep(100), send_disconnect_msg(Socket, undefined), @@ -645,7 +738,10 @@ t_publish_qos1_case03(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_publish_msg_predefined_topic(Socket, QoS, MsgId, tid(5), <<20, 21, 22, 23>>), - ?assertEqual(<<7, ?SN_PUBACK, TopicId5:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicId5:16, + MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -664,15 +760,20 @@ t_publish_qos1_case04(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_short_topic(Socket, QoS, <<"ab">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Topic = <<"ab">>, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_short_topic(Socket, QoS, MsgId, Topic, Payload1), <> = Topic, - ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, + MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), timer:sleep(100), send_disconnect_msg(Socket, undefined), @@ -692,13 +793,18 @@ t_publish_qos1_case05(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, <<"ab">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/#">>, <<20, 21, 22, 23>>), <> = <<"/#">>, - ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -724,7 +830,10 @@ t_publish_qos1_case06(_) -> send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/+">>, <<20, 21, 22, 23>>), <> = <<"/+">>, - ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -751,7 +860,11 @@ t_publish_qos2_case01(_) -> send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1), ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)), send_pubrel_msg(Socket, MsgId), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, 1:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)), timer:sleep(100), @@ -773,15 +886,21 @@ t_publish_qos2_case02(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, ?FNU:1, QoS:2, ?FNU:5, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, ?FNU:1, QoS:2, ?FNU:5, + PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId, PredefTopicId, Payload1), ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)), send_pubrel_msg(Socket, MsgId), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC :2, PredefTopicId:16, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2, + PredefTopicId:16, 1:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)), timer:sleep(100), @@ -812,7 +931,11 @@ t_publish_qos2_case03(_) -> ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)), send_pubrel_msg(Socket, MsgId), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_SHORT_TOPIC :2, <<"/a">>/binary, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2, + "/a", 1:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)), timer:sleep(100), @@ -1083,7 +1206,11 @@ t_asleep_test03_to_awake_qos1_dl_msg(_) -> send_register_msg(Socket, TopicName1, MsgId1), ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId1:16, 0:8>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, TopicId1, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ReturnCode>>, receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state send_disconnect_msg(Socket, 1), @@ -1109,7 +1236,10 @@ t_asleep_test03_to_awake_qos1_dl_msg(_) -> %% the broker should sent dl msgs to the awake client before sending the pingresp UdpData = receive_response(Socket), - MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicId1, Payload1}, UdpData), + MsgId_udp = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicId1, Payload1}, UdpData), send_puback_msg(Socket, TopicId1, MsgId_udp), %% check the pingresp is received at last @@ -1141,8 +1271,11 @@ t_asleep_test04_to_awake_qos1_dl_msg(_) -> CleanSession = 0, ReturnCode = 0, send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1,CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId1:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state send_disconnect_msg(Socket, 1), @@ -1176,11 +1309,17 @@ t_asleep_test04_to_awake_qos1_dl_msg(_) -> send_regack_msg(Socket, TopicIdNew, MsgId3), UdpData2 = receive_response(Socket), - MsgId_udp2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload1}, UdpData2), + MsgId_udp2 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload1}, UdpData2), send_puback_msg(Socket, TopicIdNew, MsgId_udp2), UdpData3 = receive_response(Socket), - MsgId_udp3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData3), + MsgId_udp3 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload2}, UdpData3), send_puback_msg(Socket, TopicIdNew, MsgId_udp3), ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), @@ -1216,8 +1355,11 @@ t_asleep_test05_to_awake_qos1_dl_msg(_) -> CleanSession = 0, ReturnCode = 0, send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId1:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state SleepDuration = 30, @@ -1250,21 +1392,28 @@ t_asleep_test05_to_awake_qos1_dl_msg(_) -> send_regack_msg(Socket, TopicIdNew, MsgId_reg), UdpData2 = receive_response(Socket), - MsgId2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2), + MsgId2 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload2}, UdpData2), send_puback_msg(Socket, TopicIdNew, MsgId2), timer:sleep(50), UdpData3 = wrap_receive_response(Socket), - MsgId3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3), + MsgId3 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload3}, UdpData3), send_puback_msg(Socket, TopicIdNew, MsgId3), timer:sleep(50), case receive_response(Socket) of <<2,23>> -> ok; UdpData4 -> - MsgId4 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, - CleanSession, ?SN_NORMAL_TOPIC, - TopicIdNew, Payload4}, UdpData4), + MsgId4 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload4}, UdpData4), send_puback_msg(Socket, TopicIdNew, MsgId4) end, ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), @@ -1322,7 +1471,10 @@ t_asleep_test06_to_awake_qos2_dl_msg(_) -> send_pingreq_msg(Socket, ClientId), UdpData = wrap_receive_response(Socket), - MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicId_tom, Payload1}, UdpData), + MsgId_udp = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicId_tom, Payload1}, UdpData), send_pubrec_msg(Socket, MsgId_udp), ?assertMatch(<<_:8, ?SN_PUBREL:8, _/binary>>, receive_response(Socket)), send_pubcomp_msg(Socket, MsgId_udp), @@ -1357,8 +1509,11 @@ t_asleep_test07_to_connected(_) -> send_register_msg(Socket, TopicName_tom, MsgId1), TopicId_tom = check_regack_msg_on_udp(MsgId1, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, TopicId_tom, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId_tom:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1,CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId_tom:16, MsgId1:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state send_disconnect_msg(Socket, SleepDuration), @@ -1436,8 +1591,11 @@ t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> CleanSession = 0, ReturnCode = 0, send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1,CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId1:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state SleepDuration = 30, send_disconnect_msg(Socket, SleepDuration), @@ -1471,7 +1629,10 @@ t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> udp_receive_timeout -> ok; UdpData2 -> - MsgId2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2), + MsgId2 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload2}, UdpData2), send_puback_msg(Socket, TopicIdNew, MsgId2) end, timer:sleep(100), @@ -1480,7 +1641,10 @@ t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> udp_receive_timeout -> ok; UdpData3 -> - MsgId3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3), + MsgId3 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload3}, UdpData3), send_puback_msg(Socket, TopicIdNew, MsgId3) end, timer:sleep(100), @@ -1489,16 +1653,18 @@ t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> udp_receive_timeout -> ok; UdpData4 -> - MsgId4 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, - CleanSession, ?SN_NORMAL_TOPIC, - TopicIdNew, Payload4}, UdpData4), + MsgId4 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload4}, UdpData4), send_puback_msg(Socket, TopicIdNew, MsgId4) end, ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), %% send PINGREQ again to enter awake state send_pingreq_msg(Socket, ClientId), - %% will not receive any buffered PUBLISH messages buffered before last awake, only receive PINGRESP here + %% will not receive any buffered PUBLISH messages buffered before last + %% awake, only receive PINGRESP here ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), gen_udp:close(Socket). @@ -1901,8 +2067,12 @@ check_dispatched_message(Dup, QoS, Retain, TopicIdType, TopicId, Payload, Socket PubMsg = receive_response(Socket), Length = 7 + byte_size(Payload), ?LOG("check_dispatched_message ~p~n", [PubMsg]), - ?LOG("expected ~p xx ~p~n", [<>, Payload]), - <> = PubMsg, + ?LOG("expected ~p xx ~p~n", + [<>, Payload]), + <> = PubMsg, case QoS of 0 -> ok; 1 -> send_puback_msg(Socket, TopicId, MsgId); @@ -1914,11 +2084,14 @@ check_dispatched_message(Dup, QoS, Retain, TopicIdType, TopicId, Payload, Socket get_udp_broadcast_address() -> "255.255.255.255". -check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, TopicType, TopicId, Payload}, UdpData) -> +check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, + CleanSession, TopicType, TopicId, Payload}, UdpData) -> <> = UdpData, ct:pal("UdpData: ~p, Payload: ~p, PayloadIn: ~p", [UdpData, Payload, PayloadIn]), Size9 = byte_size(Payload) + 7, - Eexp = <>, + Eexp = <>, ?assertEqual(Eexp, HeaderUdp), % mqtt-sn header should be same ?assertEqual(Payload, PayloadIn), % payload should be same MsgId. diff --git a/src/emqx_types.erl b/src/emqx_types.erl index 410ed5a27..3d5fb66a5 100644 --- a/src/emqx_types.erl +++ b/src/emqx_types.erl @@ -147,7 +147,7 @@ dn => binary(), atom() => term() }). --type(clientid() :: binary()|atom()). +-type(clientid() :: binary() | atom()). -type(username() :: maybe(binary())). -type(password() :: maybe(binary())). -type(peerhost() :: inet:ip_address()). From f2d99017a04bfb54d07fb30f646ab3cfcf064e59 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 9 Nov 2021 15:25:58 +0800 Subject: [PATCH 033/104] chore(exhook): update appup.src --- apps/emqx_exhook/src/emqx_exhook.appup.src | 10 ++++++++-- apps/emqx_exhook/test/emqx_exhook_demo_svr.erl | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index e78afef1e..ea8aacb7a 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -2,14 +2,20 @@ {VSN, [{"4.3.4", [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}, + {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-3]+">>, [{restart_application,emqx_exhook}]}, {<<".*">>, []} ], [{"4.3.4", [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}, + {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-3]+">>, [{restart_application,emqx_exhook}]}, {<<".*">>, []} diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl index 6e9691c3e..1d3896e14 100644 --- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl +++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl @@ -295,7 +295,7 @@ on_session_terminated(Req, Md) -> | {error, grpc_cowboy_h:error_response()}. on_message_publish(#{message := #{from := From} = Msg} = Req, Md) -> ?MODULE:in({?FUNCTION_NAME, Req}), - io:format(standard_error, "fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), + %io:format(standard_error, "fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), %% some cases for testing case From of <<"baduser">> -> From 7d06e48b4bfdf28964a2f03f469819316bd78535 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 9 Nov 2021 17:46:59 +0800 Subject: [PATCH 034/104] chore: remove needless catch Co-authored-by: Zaiming (Stone) Shi --- apps/emqx_exhook/src/emqx_exhook_server.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 9e88d774d..276f5a638 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -183,7 +183,7 @@ resovle_hookspec(HookSpecs) when is_list(HookSpecs) -> Name0 -> Name = try binary_to_existing_atom(Name0, utf8) - catch T:R:_ -> {T,R} + catch T:R -> {T,R} end, case {lists:member(Name, AvailableHooks), lists:member(Name, MessageHooks)} of From 8dfc8ed96b133776235cccdef13b534f502f6bd1 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 10 Nov 2021 16:24:42 +0800 Subject: [PATCH 035/104] chore: fill message headers --- apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl | 18 +++++++--- .../emqx_exproto/src/emqx_exproto_channel.erl | 29 ++++++++++++---- apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl | 18 ++++++++-- apps/emqx_stomp/src/emqx_stomp_protocol.erl | 34 +++++++++++++------ 4 files changed, 76 insertions(+), 23 deletions(-) diff --git a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl index d465f9ca3..89fb411b2 100644 --- a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl +++ b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl @@ -244,15 +244,26 @@ chann_publish(Topic, Payload, State = #state{clientid = ClientId}) -> case emqx_access_control:check_acl(clientinfo(State), publish, Topic) of allow -> _ = emqx_broker:publish( - emqx_message:set_flag(retain, false, - emqx_message:make(ClientId, ?QOS_0, Topic, Payload))), - ok; + packet_to_message(Topic, Payload, State)), ok; deny -> ?LOG(warning, "publish to ~p by clientid ~p failed due to acl check.", [Topic, ClientId]), {error, forbidden} end. +packet_to_message(Topic, Payload, + #state{clientid = ClientId, + username = Username, + peername = {PeerHost, _}}) -> + Message = emqx_message:set_flag( + retain, false, + emqx_message:make(ClientId, ?QOS_0, Topic, Payload) + ), + emqx_message:set_headers( + #{ proto_ver => 1 + , protocol => coap + , username => Username + , peerhost => PeerHost}, Message). %%-------------------------------------------------------------------- %% Deliver @@ -384,4 +395,3 @@ clientinfo(#state{peername = {PeerHost, _}, mountpoint => undefined, ws_cookie => undefined }. - diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 67f85f932..d8ceae4bd 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -340,17 +340,14 @@ handle_call({unsubscribe, TopicFilter}, handle_call({publish, Topic, Qos, Payload}, Channel = #channel{ conn_state = connected, - clientinfo = ClientInfo - = #{clientid := From, - mountpoint := Mountpoint}}) -> + clientinfo = ClientInfo}) -> case is_acl_enabled(ClientInfo) andalso emqx_access_control:check_acl(ClientInfo, publish, Topic) of deny -> {reply, {error, ?RESP_PERMISSION_DENY, <<"ACL deny">>}, Channel}; _ -> - Msg = emqx_message:make(From, Qos, Topic, Payload), - NMsg = emqx_mountpoint:mount(Mountpoint, Msg), - _ = emqx:publish(NMsg), + Msg = packet_to_message(Topic, Qos, Payload, Channel), + _ = emqx:publish(Msg), {reply, ok, Channel} end; @@ -419,6 +416,24 @@ is_anonymous(_AuthResult) -> false. clean_anonymous_clients() -> ets:delete(?CHAN_CONN_TAB, ?CHANMOCK(self())). +packet_to_message(Topic, Qos, Payload, + #channel{ + conninfo = #{proto_ver := ProtoVer}, + clientinfo = #{ + protocol := Protocol, + clientid := ClientId, + username := Username, + peerhost := PeerHost, + mountpoint := Mountpoint}}) -> + Msg = emqx_message:make( + ClientId, Qos, + Topic, Payload, #{}, + #{proto_ver => ProtoVer, + protocol => Protocol, + username => Username, + peerhost => PeerHost}), + emqx_mountpoint:mount(Mountpoint, Msg). + %%-------------------------------------------------------------------- %% Sub/UnSub %%-------------------------------------------------------------------- @@ -591,6 +606,8 @@ default_conninfo(ConnInfo) -> ConnInfo#{clean_start => true, clientid => undefined, username => undefined, + proto_name => undefined, + proto_ver => undefined, conn_props => #{}, connected => true, connected_at => erlang:system_time(millisecond), diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl index 34c72dcca..4a5fafdb0 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl @@ -235,8 +235,20 @@ unsubscribe(Topic, Lwm2mState = #lwm2m_state{endpoint_name = _EndpointName}) -> emqx_broker:unsubscribe(Topic), emqx_hooks:run('session.unsubscribed', [clientinfo(Lwm2mState), Topic, Opts]). -publish(Topic, Payload, Qos, EndpointName) -> - emqx_broker:publish(emqx_message:set_flag(retain, false, emqx_message:make(EndpointName, Qos, Topic, Payload))). +publish(Topic, Payload, Qos, + #lwm2m_state{ + version = ProtoVer, + peername = {PeerHost, _}, + endpoint_name = EndpointName}) -> + Message = emqx_message:set_flag( + retain, false, + emqx_message:make(EndpointName, Qos, Topic, Payload) + ), + NMessage = emqx_message:set_headers( + #{proto_ver => ProtoVer, + protocol => lwm2m, + peerhost => PeerHost}, Message), + emqx_broker:publish(NMessage). time_now() -> erlang:system_time(millisecond). @@ -281,7 +293,7 @@ do_send_to_broker(EventType, #{<<"data">> := Data} = Payload, #lwm2m_state{endpo emqx_lwm2m_cm:register_cmd(EndpointName, ReqPath, EventType, {Code, CodeMsg, Content}), NewPayload = maps:put(<<"msgType">>, EventType, Payload), Topic = uplink_topic(EventType, Lwm2mState), - publish(Topic, emqx_json:encode(NewPayload), _Qos = 0, Lwm2mState#lwm2m_state.endpoint_name). + publish(Topic, emqx_json:encode(NewPayload), _Qos = 0, Lwm2mState). %%-------------------------------------------------------------------- %% Auto Observe diff --git a/apps/emqx_stomp/src/emqx_stomp_protocol.erl b/apps/emqx_stomp/src/emqx_stomp_protocol.erl index fc211be10..3dcc6052f 100644 --- a/apps/emqx_stomp/src/emqx_stomp_protocol.erl +++ b/apps/emqx_stomp/src/emqx_stomp_protocol.erl @@ -588,15 +588,29 @@ next_ackid() -> put(ackid, AckId + 1), AckId. -make_mqtt_message(Topic, Headers, Body) -> - Msg = emqx_message:make(stomp, Topic, Body), - Headers1 = lists:foldl(fun(Key, Headers0) -> - proplists:delete(Key, Headers0) - end, Headers, [<<"destination">>, - <<"content-length">>, - <<"content-type">>, - <<"transaction">>, - <<"receipt">>]), +make_mqtt_message(Topic, Headers, Body, + #pstate{ + conninfo = #{proto_ver := ProtoVer}, + clientinfo = #{ + protocol := Protocol, + clientid := ClientId, + username := Username, + peerhost := PeerHost}}) -> + Msg = emqx_message:make( + ClientId, ?QOS_0, + Topic, Body, #{}, + #{proto_ver => ProtoVer, + protocol => Protocol, + username => Username, + peerhost => PeerHost}), + Headers1 = lists:foldl( + fun(Key, Headers0) -> + proplists:delete(Key, Headers0) + end, Headers, [<<"destination">>, + <<"content-length">>, + <<"content-type">>, + <<"transaction">>, + <<"receipt">>]), emqx_message:set_headers(#{stomp_headers => Headers1}, Msg). receipt_id(Headers) -> @@ -611,7 +625,7 @@ handle_recv_send_frame(#stomp_frame{command = <<"SEND">>, headers = Headers, bod allow -> _ = maybe_send_receipt(receipt_id(Headers), State), _ = emqx_broker:publish( - make_mqtt_message(Topic, Headers, iolist_to_binary(Body)) + make_mqtt_message(Topic, Headers, iolist_to_binary(Body), State) ), State; deny -> From f194ae65d299347a098f637a78c0141e61e7f011 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 10 Nov 2021 16:53:03 +0800 Subject: [PATCH 036/104] chore: update appup.src --- apps/emqx_coap/src/emqx_coap.app.src | 2 +- apps/emqx_coap/src/emqx_coap.appup.src | 9 +++++++++ apps/emqx_exproto/src/emqx_exproto.app.src | 2 +- apps/emqx_exproto/src/emqx_exproto.appup.src | 8 ++++++-- apps/emqx_lwm2m/src/emqx_lwm2m.app.src | 2 +- apps/emqx_lwm2m/src/emqx_lwm2m.appup.src | 6 ++++-- apps/emqx_stomp/src/emqx_stomp.app.src | 2 +- apps/emqx_stomp/src/emqx_stomp.appup.src | 10 ++++++++-- 8 files changed, 31 insertions(+), 10 deletions(-) create mode 100644 apps/emqx_coap/src/emqx_coap.appup.src diff --git a/apps/emqx_coap/src/emqx_coap.app.src b/apps/emqx_coap/src/emqx_coap.app.src index 2b5fcbb6a..dce5ef4f4 100644 --- a/apps/emqx_coap/src/emqx_coap.app.src +++ b/apps/emqx_coap/src/emqx_coap.app.src @@ -1,6 +1,6 @@ {application, emqx_coap, [{description, "EMQ X CoAP Gateway"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,gen_coap]}, diff --git a/apps/emqx_coap/src/emqx_coap.appup.src b/apps/emqx_coap/src/emqx_coap.appup.src new file mode 100644 index 000000000..b73e26be6 --- /dev/null +++ b/apps/emqx_coap/src/emqx_coap.appup.src @@ -0,0 +1,9 @@ +%% -*-: erlang -*- +{VSN, + [{"4.3.0",[ + {load_module, emqx_coap_mqtt_adapter, brutal_purge, soft_purge, []}]}, + {<<".*">>, []}], + [{"4.3.0",[ + {load_module, emqx_coap_mqtt_adapter, brutal_purge, soft_purge, []}]}, + {<<".*">>, []}] +}. diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index f7cab4c2e..0ed54e6fd 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.4"}, %% 4.3.3 is used by ee + {vsn, "4.3.5"}, %% 4.3.3 is used by ee {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index e0a021af5..7da28e773 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,6 +1,8 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.3", + [{"4.3.4", + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + {"4.3.3", [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {"4.3.2", @@ -12,7 +14,9 @@ {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.3", + [{"4.3.4", + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + {"4.3.3", [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {"4.3.2", diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src index 551cf8d07..b929ad854 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src @@ -1,6 +1,6 @@ {application,emqx_lwm2m, [{description,"EMQ X LwM2M Gateway"}, - {vsn, "4.3.4"}, % strict semver, bump manually! + {vsn, "4.3.5"}, % strict semver, bump manually! {modules,[]}, {registered,[emqx_lwm2m_sup]}, {applications,[kernel,stdlib,lwm2m_coap]}, diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src index 600cf236b..0cd98db0d 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src @@ -1,5 +1,5 @@ %% -*-: erlang -*- -{"4.3.4", +{VSN, [ {<<"4\\.3\\.[0-1]">>, [ {restart_application, emqx_lwm2m} @@ -7,7 +7,9 @@ {"4.3.2", [ {load_module, emqx_lwm2m_message, brutal_purge, soft_purge, []} ]}, - {"4.3.3", []} %% only config change + {<<"4\\.3\\.[3-4]">>, [ + {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} + ]} ], [ {<<"4\\.3\\.[0-1]">>, [ diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index d2ecae53b..c2f4b57d3 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -1,6 +1,6 @@ {application, emqx_stomp, [{description, "EMQ X Stomp Protocol Plugin"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_stomp_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.appup.src b/apps/emqx_stomp/src/emqx_stomp.appup.src index bf4603e52..dce441b3c 100644 --- a/apps/emqx_stomp/src/emqx_stomp.appup.src +++ b/apps/emqx_stomp/src/emqx_stomp.appup.src @@ -1,10 +1,16 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.1",[{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, + [{"4.3.2",[{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}]}, + {"4.3.1",[ + {load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, + {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{restart_application,emqx_stomp}]}, {<<".*">>,[]}], - [{"4.3.1",[{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, + [{"4.3.2",[{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}]}, + {"4.3.1",[ + {load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, + {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{restart_application,emqx_stomp}]}, {<<".*">>,[]}]}. From 2be33b33e3d9ef36bab0ab2e87985b62d2c0fd48 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 10 Nov 2021 17:05:27 +0800 Subject: [PATCH 037/104] chore: fix elvis warnings --- apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl | 4 +- .../emqx_exproto/src/emqx_exproto_channel.erl | 2 - apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl | 47 ++++++++++++------- apps/emqx_stomp/src/emqx_stomp_protocol.erl | 34 ++++++++++---- 4 files changed, 57 insertions(+), 30 deletions(-) diff --git a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl index 89fb411b2..f7f0b4eda 100644 --- a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl +++ b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl @@ -139,7 +139,7 @@ handle_call({subscribe, Topic, CoapPid}, _From, State=#state{sub_topics = TopicL NewTopics = proplists:delete(Topic, TopicList), IsWild = emqx_topic:wildcard(Topic), {reply, chann_subscribe(Topic, State), State#state{sub_topics = - [{Topic, {IsWild, CoapPid}}|NewTopics]}, hibernate}; + [{Topic, {IsWild, CoapPid}} | NewTopics]}, hibernate}; handle_call({unsubscribe, Topic, _CoapPid}, _From, State=#state{sub_topics = TopicList}) -> NewTopics = proplists:delete(Topic, TopicList), @@ -281,7 +281,7 @@ do_deliver({Topic, Payload}, Subscribers) -> deliver_to_coap(_TopicName, _Payload, []) -> ok; -deliver_to_coap(TopicName, Payload, [{TopicFilter, {IsWild, CoapPid}}|T]) -> +deliver_to_coap(TopicName, Payload, [{TopicFilter, {IsWild, CoapPid}} | T]) -> Matched = case IsWild of true -> emqx_topic:match(TopicName, TopicFilter); false -> TopicName =:= TopicFilter diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index d8ceae4bd..44b8b64d3 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -606,8 +606,6 @@ default_conninfo(ConnInfo) -> ConnInfo#{clean_start => true, clientid => undefined, username => undefined, - proto_name => undefined, - proto_ver => undefined, conn_props => #{}, connected => true, connected_at => erlang:system_time(millisecond), diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl index 4a5fafdb0..7f40041a5 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl @@ -74,7 +74,8 @@ call(Pid, Msg, Timeout) -> Error -> {error, Error} end. -init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, RegInfo = #{<<"lt">> := LifeTime, <<"lwm2m">> := Ver}) -> +init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, + RegInfo = #{<<"lt">> := LifeTime, <<"lwm2m">> := Ver}) -> Mountpoint = proplists:get_value(mountpoint, lwm2m_coap_responder:options(), ""), Lwm2mState = #lwm2m_state{peername = Peername, endpoint_name = EndpointName, @@ -103,9 +104,10 @@ init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, RegInfo = #{<<"lt">> emqx_cm:register_channel(EndpointName, CoapPid, conninfo(Lwm2mState1)) end), emqx_cm:insert_channel_info(EndpointName, info(Lwm2mState1), stats(Lwm2mState1)), - emqx_lwm2m_cm:register_channel(EndpointName, RegInfo, LifeTime, Ver, Peername), + emqx_lwm2m_cm:register_channel(EndpointName, RegInfo, LifeTime, Ver, Peername), - {ok, Lwm2mState1#lwm2m_state{life_timer = emqx_lwm2m_timer:start_timer(LifeTime, {life_timer, expired})}}; + NTimer = emqx_lwm2m_timer:start_timer(LifeTime, {life_timer, expired}), + {ok, Lwm2mState1#lwm2m_state{life_timer = NTimer}}; {error, Error} -> _ = run_hooks('client.connack', [conninfo(Lwm2mState), not_authorized], undefined), {error, Error} @@ -133,7 +135,7 @@ update_reg_info(NewRegInfo, Lwm2mState=#lwm2m_state{life_timer = LifeTimer, regi %% - report the registration info update, but only when objectList is updated. case NewRegInfo of #{<<"objectList">> := _} -> - emqx_lwm2m_cm:update_reg_info(Epn, NewRegInfo), + emqx_lwm2m_cm:update_reg_info(Epn, NewRegInfo), send_to_broker(<<"update">>, #{<<"data">> => UpdatedRegInfo}, Lwm2mState); _ -> ok end @@ -186,7 +188,8 @@ deliver(#message{topic = Topic, payload = Payload}, started_at = StartedAt, endpoint_name = EndpointName}) -> IsCacheMode = is_cache_mode(RegInfo, StartedAt), - ?LOG(debug, "Get MQTT message from broker, IsCacheModeNow?: ~p, Topic: ~p, Payload: ~p", [IsCacheMode, Topic, Payload]), + ?LOG(debug, "Get MQTT message from broker, IsCacheModeNow?: ~p, " + "Topic: ~p, Payload: ~p", [IsCacheMode, Topic, Payload]), AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), deliver_to_coap(AlternatePath, Payload, CoapPid, IsCacheMode, EndpointName), Lwm2mState. @@ -256,7 +259,8 @@ time_now() -> erlang:system_time(millisecond). %% Deliver downlink message to coap %%-------------------------------------------------------------------- -deliver_to_coap(AlternatePath, JsonData, CoapPid, CacheMode, EndpointName) when is_binary(JsonData)-> +deliver_to_coap(AlternatePath, JsonData, + CoapPid, CacheMode, EndpointName) when is_binary(JsonData)-> try TermData = emqx_json:decode(JsonData, [return_maps]), deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode, EndpointName) @@ -285,7 +289,8 @@ deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode, EndpointName) when send_to_broker(EventType, Payload = #{}, Lwm2mState) -> do_send_to_broker(EventType, Payload, Lwm2mState). -do_send_to_broker(EventType, #{<<"data">> := Data} = Payload, #lwm2m_state{endpoint_name = EndpointName} = Lwm2mState) -> +do_send_to_broker(EventType, #{<<"data">> := Data} = Payload, + #lwm2m_state{endpoint_name = EndpointName} = Lwm2mState) -> ReqPath = maps:get(<<"reqPath">>, Data, undefined), Code = maps:get(<<"code">>, Data, undefined), CodeMsg = maps:get(<<"codeMsg">>, Data, undefined), @@ -327,18 +332,27 @@ auto_observe(AlternatePath, ObjectList, CoapPid, EndpointName) -> observe_object_list(AlternatePath, ObjectList, CoapPid, EndpointName) -> lists:foreach(fun(ObjectPath) -> - [ObjId| LastPath] = emqx_lwm2m_cmd_handler:path_list(ObjectPath), + [ObjId | LastPath] = emqx_lwm2m_cmd_handler:path_list(ObjectPath), case ObjId of <<"19">> -> [ObjInsId | _LastPath1] = LastPath, case ObjInsId of <<"0">> -> - observe_object_slowly(AlternatePath, <<"/19/0/0">>, CoapPid, 100, EndpointName); + observe_object_slowly( + AlternatePath, <<"/19/0/0">>, + CoapPid, 100, EndpointName + ); _ -> - observe_object_slowly(AlternatePath, ObjectPath, CoapPid, 100, EndpointName) + observe_object_slowly( + AlternatePath, ObjectPath, + CoapPid, 100, EndpointName + ) end; _ -> - observe_object_slowly(AlternatePath, ObjectPath, CoapPid, 100, EndpointName) + observe_object_slowly( + AlternatePath, ObjectPath, + CoapPid, 100, EndpointName + ) end end, ObjectList). @@ -392,11 +406,12 @@ get_cached_downlink_messages() -> is_cache_mode(RegInfo, StartedAt) -> case is_psm(RegInfo) orelse is_qmode(RegInfo) of true -> - QModeTimeWind = proplists:get_value(qmode_time_window, lwm2m_coap_responder:options(), 22), - Now = time_now(), - if (Now - StartedAt) >= QModeTimeWind -> true; - true -> false - end; + QModeTimeWind = proplists:get_value( + qmode_time_window, + lwm2m_coap_responder:options(), + 22 + ), + (time_now() - StartedAt) >= QModeTimeWind; false -> false end. diff --git a/apps/emqx_stomp/src/emqx_stomp_protocol.erl b/apps/emqx_stomp/src/emqx_stomp_protocol.erl index 3dcc6052f..4e371d9b0 100644 --- a/apps/emqx_stomp/src/emqx_stomp_protocol.erl +++ b/apps/emqx_stomp/src/emqx_stomp_protocol.erl @@ -108,6 +108,8 @@ , init/2 ]}). +-elvis([{elvis_style, dont_repeat_yourself, disable}]). + -type(pstate() :: #pstate{}). %% @doc Init protocol @@ -132,8 +134,7 @@ init(ConnInfo = #{peername := {PeerHost, _Port}, AllowAnonymous = get_value(allow_anonymous, Opts, false), DefaultUser = get_value(default_user, Opts), - - #pstate{ + #pstate{ conninfo = NConnInfo, clientinfo = ClientInfo, heartfun = HeartFun, @@ -165,7 +166,7 @@ default_conninfo(ConnInfo) -> info(State) -> maps:from_list(info(?INFO_KEYS, State)). --spec info(list(atom())|atom(), pstate()) -> term(). +-spec info(list(atom()) | atom(), pstate()) -> term(). info(Keys, State) when is_list(Keys) -> [{Key, info(Key, State)} || Key <- Keys]; info(conninfo, #pstate{conninfo = ConnInfo}) -> @@ -288,7 +289,12 @@ received(#stomp_frame{command = <<"CONNECT">>}, State = #pstate{connected = true received(Frame = #stomp_frame{command = <<"SEND">>, headers = Headers}, State) -> case header(<<"transaction">>, Headers) of undefined -> {ok, handle_recv_send_frame(Frame, State)}; - TransactionId -> add_action(TransactionId, {fun ?MODULE:handle_recv_send_frame/2, [Frame]}, receipt_id(Headers), State) + TransactionId -> + add_action(TransactionId, + {fun ?MODULE:handle_recv_send_frame/2, [Frame]}, + receipt_id(Headers), + State + ) end; received(#stomp_frame{command = <<"SUBSCRIBE">>, headers = Headers}, @@ -346,7 +352,11 @@ received(#stomp_frame{command = <<"UNSUBSCRIBE">>, headers = Headers}, received(Frame = #stomp_frame{command = <<"ACK">>, headers = Headers}, State) -> case header(<<"transaction">>, Headers) of undefined -> {ok, handle_recv_ack_frame(Frame, State)}; - TransactionId -> add_action(TransactionId, {fun ?MODULE:handle_recv_ack_frame/2, [Frame]}, receipt_id(Headers), State) + TransactionId -> + add_action(TransactionId, + {fun ?MODULE:handle_recv_ack_frame/2, [Frame]}, + receipt_id(Headers), + State) end; %% NACK @@ -357,7 +367,11 @@ received(Frame = #stomp_frame{command = <<"ACK">>, headers = Headers}, State) -> received(Frame = #stomp_frame{command = <<"NACK">>, headers = Headers}, State) -> case header(<<"transaction">>, Headers) of undefined -> {ok, handle_recv_nack_frame(Frame, State)}; - TransactionId -> add_action(TransactionId, {fun ?MODULE:handle_recv_nack_frame/2, [Frame]}, receipt_id(Headers), State) + TransactionId -> + add_action(TransactionId, + {fun ?MODULE:handle_recv_nack_frame/2, [Frame]}, + receipt_id(Headers), + State) end; %% BEGIN @@ -516,9 +530,9 @@ negotiate_version(Accepts) -> negotiate_version(Ver, []) -> {error, <<"Supported protocol versions < ", Ver/binary>>}; -negotiate_version(Ver, [AcceptVer|_]) when Ver >= AcceptVer -> +negotiate_version(Ver, [AcceptVer | _]) when Ver >= AcceptVer -> {ok, AcceptVer}; -negotiate_version(Ver, [_|T]) -> +negotiate_version(Ver, [_ | T]) -> negotiate_version(Ver, T). check_login(Login, _, AllowAnonymous, _) @@ -537,7 +551,7 @@ check_login(Login, Passcode, _, DefaultUser) -> add_action(Id, Action, ReceiptId, State = #pstate{transaction = Trans}) -> case maps:get(Id, Trans, undefined) of {Ts, Actions} -> - NTrans = Trans#{Id => {Ts, [Action|Actions]}}, + NTrans = Trans#{Id => {Ts, [Action | Actions]}}, {ok, State#pstate{transaction = NTrans}}; _ -> send(error_frame(ReceiptId, ["Transaction ", Id, " not found"]), State) @@ -713,7 +727,7 @@ find_sub_by_id(Id, Subs) -> end, Subs), case maps:to_list(Found) of [] -> undefined; - [Sub|_] -> Sub + [Sub | _] -> Sub end. is_acl_enabled(_) -> From 6b40048d297f8f6d4728eef4dcc89e7bb8344565 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 10 Nov 2021 18:40:40 +0800 Subject: [PATCH 038/104] chore: put the pool_size default value to avoid hot upgrade failure --- apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl | 6 ++++-- apps/emqx_exhook/src/emqx_exhook_mngr.erl | 4 +++- apps/emqx_exhook/test/emqx_exhook_demo_svr.erl | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl index f7f0b4eda..a6ea09881 100644 --- a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl +++ b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl @@ -58,6 +58,8 @@ -define(SUBOPTS, #{rh => 0, rap => 0, nl => 0, qos => ?QOS_0, is_new => false}). +-define(PROTO_VER, 1). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -260,7 +262,7 @@ packet_to_message(Topic, Payload, emqx_message:make(ClientId, ?QOS_0, Topic, Payload) ), emqx_message:set_headers( - #{ proto_ver => 1 + #{ proto_ver => ?PROTO_VER , protocol => coap , username => Username , peerhost => PeerHost}, Message). @@ -335,7 +337,7 @@ conninfo(#state{peername = Peername, peercert => nossl, %% TODO: dtls conn_mod => ?MODULE, proto_name => <<"CoAP">>, - proto_ver => 1, + proto_ver => ?PROTO_VER, clean_start => true, clientid => ClientId, username => undefined, diff --git a/apps/emqx_exhook/src/emqx_exhook_mngr.erl b/apps/emqx_exhook/src/emqx_exhook_mngr.erl index 0f3c8b8ab..d4c493cb8 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mngr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mngr.erl @@ -295,7 +295,9 @@ put_pool_size(Val) -> persistent_term:put({?APP, pool_size}, Val). get_pool_size() -> - persistent_term:get({?APP, pool_size}). + %% Avoid the scenario that the parameter is not set after + %% the hot upgrade completed. + persistent_term:get({?APP, pool_size}, erlang:system_info(schedulers)). save(Name, ServerState) -> Saved = persistent_term:get(?APP, []), diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl index 1d3896e14..b05748856 100644 --- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl +++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl @@ -295,7 +295,7 @@ on_session_terminated(Req, Md) -> | {error, grpc_cowboy_h:error_response()}. on_message_publish(#{message := #{from := From} = Msg} = Req, Md) -> ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format(standard_error, "fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), + %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), %% some cases for testing case From of <<"baduser">> -> From 08cf0326b318a589fb082fab0f0bc975548bcfe8 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 10 Nov 2021 19:18:42 +0800 Subject: [PATCH 039/104] chore(exhook): bump version to 4.4.0 --- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook.appup.src | 20 ++------------------ 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index b386bcaca..715060df4 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,6 +1,6 @@ {application, emqx_exhook, [{description, "EMQ X Extension for Hook"}, - {vsn, "4.3.5"}, + {vsn, "4.4.0"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index ea8aacb7a..0fe0ea78f 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -1,23 +1,7 @@ %% -*-: erlang -*- {VSN, - [{"4.3.4", - [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}, - {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[0-3]+">>, - [{restart_application,emqx_exhook}]}, - {<<".*">>, []} + [{<<".*">>, []} ], - [{"4.3.4", - [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}, - {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[0-3]+">>, - [{restart_application,emqx_exhook}]}, - {<<".*">>, []} + [{<<".*">>, []} ] }. From f7bdd6defe43d3df4d553c9fc321e467f1bc423e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 12 Nov 2021 15:35:08 +0800 Subject: [PATCH 040/104] chore(lwm2m): fix bad appup.src --- apps/emqx_lwm2m/src/emqx_lwm2m.appup.src | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src index 0cd98db0d..6656eb149 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src @@ -7,7 +7,8 @@ {"4.3.2", [ {load_module, emqx_lwm2m_message, brutal_purge, soft_purge, []} ]}, - {<<"4\\.3\\.[3-4]">>, [ + {"4.3.3", []}, %% only config change + {"4.3.4", [ {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} ]} ], @@ -18,6 +19,9 @@ {"4.3.2", [ {load_module, emqx_lwm2m_message, brutal_purge, soft_purge, []} ]}, - {"4.3.3", []} %% only config change + {"4.3.3", []}, %% only config change + {"4.3.4", [ + {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} + ]} ] }. From 25f504c90ac27d7685e598aab179c2885b033efe Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 12 Nov 2021 15:59:35 +0800 Subject: [PATCH 041/104] feat(mongo srv): support srv for mongodb authentication --- apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf | 9 +- .../priv/emqx_auth_mongo.schema | 37 +++++--- .../src/emqx_auth_mongo.app.src | 2 +- .../src/emqx_auth_mongo_sup.erl | 93 ++++++++++++++++++- 4 files changed, 121 insertions(+), 20 deletions(-) diff --git a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf index 2a3d038f0..3d3d0ee5b 100644 --- a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf +++ b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf @@ -5,7 +5,13 @@ ## MongoDB Topology Type. ## ## Value: single | unknown | sharded | rs -auth.mongo.type = single +auth.mongo.type = + +## Whether to use SRV and TXT records. +## +## Value: true | false +## Default: false +auth.mongo.srv_record = false ## The set name if type is rs. ## @@ -37,7 +43,6 @@ auth.mongo.pool = 8 ## MongoDB AuthSource ## ## Value: String -## Default: mqtt ## auth.mongo.auth_source = admin ## MongoDB database diff --git a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema index 8a2ff98b3..17a83c37c 100644 --- a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema +++ b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema @@ -6,8 +6,12 @@ {datatype, {enum, [single, unknown, sharded, rs]}} ]}. +{mapping, "auth.mongo.srv_record", "emqx_auth_mongo.server", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + {mapping, "auth.mongo.rs_set_name", "emqx_auth_mongo.server", [ - {default, "mqtt"}, {datatype, string} ]}. @@ -41,7 +45,6 @@ ]}. {mapping, "auth.mongo.auth_source", "emqx_auth_mongo.server", [ - {default, "mqtt"}, {datatype, string} ]}. @@ -101,9 +104,9 @@ ]}. {translation, "emqx_auth_mongo.server", fun(Conf) -> - H = cuttlefish:conf_get("auth.mongo.server", Conf), - Hosts = string:tokens(H, ","), - Type0 = cuttlefish:conf_get("auth.mongo.type", Conf), + SrvRecord = cuttlefish:conf_get("auth.mongo.srv_record", Conf, false), + Server = cuttlefish:conf_get("auth.mongo.server", Conf), + Type = cuttlefish:conf_get("auth.mongo.type", Conf), Pool = cuttlefish:conf_get("auth.mongo.pool", Conf), %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 Login = cuttlefish:conf_get("auth.mongo.username", Conf, @@ -111,7 +114,10 @@ ), Passwd = cuttlefish:conf_get("auth.mongo.password", Conf), DB = cuttlefish:conf_get("auth.mongo.database", Conf), - AuthSrc = cuttlefish:conf_get("auth.mongo.auth_source", Conf), + AuthSource = case cuttlefish:conf_get("auth.mongo.auth_source", Conf, undefined) of + undefined -> []; + AuthSource0 -> [{auth_source, list_to_binary(AuthSource0)}] + end, R = cuttlefish:conf_get("auth.mongo.w_mode", Conf), W = cuttlefish:conf_get("auth.mongo.r_mode", Conf), Login0 = case Login =:= [] of @@ -156,8 +162,8 @@ false -> [] end, - WorkerOptions = [{database, list_to_binary(DB)}, {auth_source, list_to_binary(AuthSrc)}] - ++ Login0 ++ Passwd0 ++ W0 ++ R0 ++ Ssl, + WorkerOptions = [{database, list_to_binary(DB)}] + ++ Login0 ++ Passwd0 ++ W0 ++ R0 ++ Ssl ++ AuthSource, Vars = cuttlefish_variable:fuzzy_matches(["auth", "mongo", "topology", "$name"], Conf), Options = lists:map(fun({_, Name}) -> @@ -174,16 +180,17 @@ {list_to_atom(Name2), cuttlefish:conf_get("auth.mongo.topology."++Name, Conf)} end, Vars), - Type = case Type0 =:= rs of - true -> {Type0, list_to_binary(cuttlefish:conf_get("auth.mongo.rs_set_name", Conf))}; - false -> Type0 - end, - [{type, Type}, - {hosts, Hosts}, + ReplicaSet = case cuttlefish:conf_get("auth.mongo.rs_set_name", Conf, undefined) of + undefined -> []; + ReplicaSet0 -> [{rs_set_name, list_to_binary(ReplicaSet0)}] + end, + [{srv_record, SrvRecord}, + {type, Type}, + {server, Server}, {options, Options}, {worker_options, WorkerOptions}, {auto_reconnect, 1}, - {pool_size, Pool}] + {pool_size, Pool}] ++ ReplicaSet end}. %% The mongodb operation timeout is specified by the value of `cursor_timeout` from application config, diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src index cc4e72ef3..ab0b4ff56 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mongo, [{description, "EMQ X Authentication/ACL with MongoDB"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mongo_sup]}, {applications, [kernel,stdlib,mongodb,ecpool]}, diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl b/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl index 3f27cb1dd..55263494a 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl @@ -28,7 +28,96 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - {ok, PoolEnv} = application:get_env(?APP, server), - PoolSpec = ecpool:pool_spec(?APP, ?APP, ?APP, PoolEnv), + {ok, Opts} = application:get_env(?APP, server), + NOpts = may_parse_srv_and_txt_records(Opts), + PoolSpec = ecpool:pool_spec(?APP, ?APP, ?APP, NOpts), {ok, {{one_for_all, 10, 100}, [PoolSpec]}}. +may_parse_srv_and_txt_records(Opts) when is_list(Opts) -> + maps:to_list(may_parse_srv_and_txt_records(maps:from_list(Opts))); + +may_parse_srv_and_txt_records(#{type := Type, + srv_record := false, + server := Server} = Opts) -> + Hosts = to_hosts(Server), + case Type =:= rs of + true -> + case maps:get(rs_set_name, Opts, undefined) of + undefined -> + error({missing_parameter, rs_set_name}); + ReplicaSet -> + Opts#{type => {rs, ReplicaSet}, + hosts => Hosts} + end; + false -> + Opts#{hosts => Hosts} + end; + +may_parse_srv_and_txt_records(#{type := Type, + srv_record := true, + server := Server, + worker_options := WorkerOptions} = Opts) -> + Hosts = parse_srv_records(Server), + Opts0 = parse_txt_records(Type, Server), + NWorkerOptions = maps:to_list(maps:merge(maps:from_list(WorkerOptions), maps:with([auth_source], Opts0))), + NOpts = Opts#{hosts => Hosts, worker_options => NWorkerOptions}, + case Type =:= rs of + true -> + case maps:get(rs_set_name, Opts0, maps:get(rs_set_name, NOpts, undefined)) of + undefined -> + error({missing_parameter, rs_set_name}); + ReplicaSet -> + NOpts#{type => {Type, ReplicaSet}} + end; + false -> + NOpts + end. + +to_hosts(Server) -> + [string:trim(H) || H <- string:tokens(Server, ",")]. + +parse_srv_records(Server) -> + case inet_res:lookup("_mongodb._tcp." ++ Server, in, srv) of + [] -> + error(service_not_found); + Services -> + [Host ++ ":" ++ integer_to_list(Port) || {_, _, Port, Host} <- Services] + end. + +parse_txt_records(Type, Server) -> + case inet_res:lookup(Server, in, txt) of + [] -> + #{}; + [[QueryString]] -> + case uri_string:dissect_query(QueryString) of + {error, _, _} -> + error({invalid_txt_record, invalid_query_string}); + Options -> + Fields = case Type of + rs -> ["authSource", "replicaSet"]; + _ -> ["authSource"] + end, + take_and_convert(Fields, Options) + end; + _ -> + error({invalid_txt_record, multiple_records}) + end. + +take_and_convert(Fields, Options) -> + take_and_convert(Fields, Options, #{}). + +take_and_convert([], [_ | _], _Acc) -> + error({invalid_txt_record, invalid_option}); +take_and_convert([], [], Acc) -> + Acc; +take_and_convert([Field | More], Options, Acc) -> + case lists:keytake(Field, 1, Options) of + {value, {"authSource", V}, NOptions} -> + take_and_convert(More, NOptions, Acc#{auth_source => list_to_binary(V)}); + {value, {"replicaSet", V}, NOptions} -> + take_and_convert(More, NOptions, Acc#{rs_set_name => list_to_binary(V)}); + {value, _, _} -> + error({invalid_txt_record, invalid_option}); + false -> + take_and_convert(More, Options, Acc) + end. From cb185389577263c17af0f3348b82c4f3fc50464a Mon Sep 17 00:00:00 2001 From: zhouzb Date: Mon, 15 Nov 2021 09:41:03 +0800 Subject: [PATCH 042/104] fix(mong srv): fix wrong configuration --- apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf index 3d3d0ee5b..8baddae19 100644 --- a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf +++ b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf @@ -5,7 +5,7 @@ ## MongoDB Topology Type. ## ## Value: single | unknown | sharded | rs -auth.mongo.type = +auth.mongo.type = single ## Whether to use SRV and TXT records. ## From 454f609aa6cb057772f56fdb7533ab9036ebd2ed Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 12 Nov 2021 14:18:32 +0800 Subject: [PATCH 043/104] build: emqx package name scheme include otp version --- Makefile | 1 + build | 5 ++++- deploy/packages/deb/Makefile | 2 +- deploy/packages/rpm/Makefile | 2 +- scripts/get-otp-vsn.sh | 7 +++++++ 5 files changed, 14 insertions(+), 3 deletions(-) create mode 100755 scripts/get-otp-vsn.sh diff --git a/Makefile b/Makefile index 17ceb9481..11edd8929 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ REBAR_VERSION = 3.14.3-emqx-8 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts +export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X export EMQX_CE_DASHBOARD_VERSION ?= v4.3.3 diff --git a/build b/build index be4f88672..dbbfcbdfb 100755 --- a/build +++ b/build @@ -12,6 +12,9 @@ ARTIFACT="$2" # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" +OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" +export OTP_VSN + PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" export PKG_VSN @@ -116,7 +119,7 @@ make_zip() { log "ERROR: $tarball is not found" fi local zipball - zipball="${pkgpath}/${PROFILE}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip" + zipball="${pkgpath}/${PROFILE}-${OTP_VSN}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip" tar zxf "${tarball}" -C "${tard}/emqx" ## try to be portable for zip packages. ## for DEB and RPM packages the dependencies are resoved by yum and apt diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 48124d780..6f3956409 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -8,7 +8,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(OTP_VSN)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) .PHONY: all all: | $(BUILT) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 5a6e6bee4..e8c01e935 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -16,7 +16,7 @@ endif EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz -TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(OTP_VSN)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) ifeq ($(RPM_REL),) # no tail RPM_REL := 1 diff --git a/scripts/get-otp-vsn.sh b/scripts/get-otp-vsn.sh new file mode 100755 index 000000000..fd5b4fd0b --- /dev/null +++ b/scripts/get-otp-vsn.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euo pipefail + +vsn=$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell) + +echo "otp${vsn}" From 7e91a47be8b1ae68ded8d7ddb22350be8a8eebc9 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 12 Nov 2021 14:50:56 +0800 Subject: [PATCH 044/104] ci: update workflows --- .github/workflows/build_packages.yaml | 53 +++++++++++---------------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index befa37912..efaf14bfc 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -212,6 +212,8 @@ jobs: strategy: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + otp: + - 23.3.4.8-1 arch: - amd64 - arm64 @@ -248,15 +250,11 @@ jobs: shell: bash steps: - - name: prepare docker - run: | - mkdir -p $HOME/.docker - echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json - echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50}' | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker - docker info - docker buildx create --use --name mybuild - docker run --rm --privileged tonistiigi/binfmt --install all + - uses: docker/setup-buildx-action@v1 + - uses: docker/setup-qemu-action@v1 + with: + image: tonistiigi/binfmt:latest + platforms: all - uses: actions/download-artifact@v2 with: name: source @@ -265,6 +263,7 @@ jobs: run: unzip -q source.zip - name: downloads old emqx zip packages env: + OTP_VSN: otp${{ matrix.otp }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} @@ -283,36 +282,28 @@ jobs: cd source/_upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for tag in ${old_vsns[@]}; do - if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256 - echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1 + if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip) | grep -oE "^[23]+")" ];then + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip.sha256 + echo "$(cat ${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip.sha256) ${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip" | sha256sum -c || exit 1 fi done - name: build emqx packages env: - ERL_OTP: erl23.2.7.2-emqx-2 + OTP: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} + working-directory: source run: | - set -e -u - cd source - docker buildx build --no-cache \ - --platform=linux/$ARCH \ - -t cross_build_emqx_for_$SYSTEM \ - -f .ci/build_packages/Dockerfile \ - --build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-$SYSTEM \ - --build-arg EMQX_NAME=$PROFILE \ - --output type=tar,dest=/tmp/cross-build-$PROFILE-for-$SYSTEM.tar . - - mkdir -p /tmp/packages/$PROFILE - tar -xvf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar --wildcards emqx/_packages/$PROFILE/* - mv emqx/_packages/$PROFILE/* /tmp/packages/$PROFILE/ - rm -rf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar - - docker rm -f $(docker ps -a -q) - docker volume prune -f + docker run -i --rm \ + -v $(pwd):/emqx \ + --workdir /emqx \ + --platform linux/$ARCH \ + ghcr.io/emqx/emqx-builder/4.4:$OTP-$SYSTEM \ + bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \ + make $PROFILE-pkg || cat rebar3.crashdump; \ + EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" - name: create sha256 env: PROFILE: ${{ matrix.profile}} From f46084438b1994084137dd07a4322ebf20a1084e Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 28 Oct 2021 09:19:30 +0800 Subject: [PATCH 045/104] chore(cluster): add new type for dns auto cluster Signed-off-by: zhanghongtong --- etc/emqx.conf | 5 +++++ priv/emqx.schema | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 7b81e40cc..78d383930 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -101,6 +101,11 @@ cluster.autoclean = 5m ## Value: String ## cluster.dns.app = emqx +## Type of dns record. +## +## Value: Value: a | srv +## cluster.dns.type = a + ##-------------------------------------------------------------------- ## Cluster using etcd diff --git a/priv/emqx.schema b/priv/emqx.schema index 602de56b9..f8df59da9 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -96,6 +96,11 @@ {datatype, string} ]}. +{mapping, "cluster.dns.type", "ekka.cluster_discovery", [ + {datatype, {enum, [a, srv]}}, + {default, a} +]}. + %%-------------------------------------------------------------------- %% Cluster using etcd @@ -171,7 +176,8 @@ {loop, cuttlefish:conf_get("cluster.mcast.loop", Conf, true)}]; (dns) -> [{name, cuttlefish:conf_get("cluster.dns.name", Conf)}, - {app, cuttlefish:conf_get("cluster.dns.app", Conf)}]; + {app, cuttlefish:conf_get("cluster.dns.app", Conf)}, + {type, cuttlefish:conf_get("cluster.dns.type", Conf)}]; (etcd) -> SslOpts = fun(Conf) -> Options = cuttlefish_variable:filter_by_prefix("cluster.etcd.ssl", Conf), From 87a2667e35ec6ee62b8918f4d66c04f4f2b2e938 Mon Sep 17 00:00:00 2001 From: lafirest Date: Mon, 15 Nov 2021 10:57:59 +0800 Subject: [PATCH 046/104] fix(emqx_retainer): fix timer message error (#6156) * fix(emqx_retainer): fix timer message error --- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- apps/emqx_retainer/src/emqx_retainer.erl | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 4ef423b78..a423fb9b7 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQ X Retainer"}, - {vsn, "4.4.1"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index fcc5652cc..b6de9d9f6 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -156,7 +156,7 @@ start_expire_timer(0, State) -> start_expire_timer(undefined, State) -> State; start_expire_timer(Ms, State) -> - Timer = erlang:send_after(Ms, self(), stats), + Timer = erlang:send_after(Ms, self(), {expire, Ms}), State#state{expiry_timer = Timer}. handle_call(Req, _From, State) -> @@ -168,12 +168,14 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info(stats, State = #state{stats_fun = StatsFun}) -> + StatsTimer = erlang:send_after(timer:seconds(1), self(), stats), StatsFun(retained_count()), - {noreply, State, hibernate}; + {noreply, State#state{stats_timer = StatsTimer}, hibernate}; -handle_info(expire, State) -> +handle_info({expire, Ms} = Expire, State) -> + Timer = erlang:send_after(Ms, self(), Expire), ok = expire_messages(), - {noreply, State, hibernate}; + {noreply, State#state{expiry_timer = Timer}, hibernate}; handle_info(Info, State) -> ?LOG(error, "Unexpected info: ~p", [Info]), From 0357f7ad853db6b83b1c92978885dacd3cbbb2e4 Mon Sep 17 00:00:00 2001 From: lafirest Date: Mon, 15 Nov 2021 11:00:04 +0800 Subject: [PATCH 047/104] improve(emqx_st_statistics): optimize the parameters of on_publish_done (#6151) * fix(emqx_st_statistics): optimize the parameters of on_publish_done --- .../emqx_st_statistics/emqx_st_statistics.erl | 35 +++++++++++++------ src/emqx_session.erl | 9 +++-- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl index 21cf2692e..f22aec41c 100644 --- a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl +++ b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl @@ -24,7 +24,7 @@ -logger_header("[SLOW TOPICS]"). --export([ start_link/1, on_publish_done/5, enable/0 +-export([ start_link/1, on_publish_done/3, enable/0 , disable/0, clear_history/0 ]). @@ -42,7 +42,7 @@ -type state() :: #{ config := proplist:proplist() , period := pos_integer() , last_tick_at := pos_integer() - , counter := counters:counter_ref() + , counter := counters:counters_ref() , enable := boolean() }. @@ -70,6 +70,13 @@ -type slow_log() :: #slow_log{}. -type top_k_map() :: #{emqx_types:topic() => top_k()}. +-type publish_done_env() :: #{ ignore_before_create := boolean() + , threshold := pos_integer() + , counter := counters:counters_ref() + }. + +-type publish_done_args() :: #{session_rebirth_time => pos_integer()}. + -ifdef(TEST). -define(TOPK_ACCESS, public). -else. @@ -90,13 +97,16 @@ start_link(Env) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). --spec on_publish_done(message(), - pos_integer(), boolean(), pos_integer(), counters:counters_ref()) -> ok. -on_publish_done(#message{timestamp = Timestamp}, Created, IgnoreBeforeCreate, _, _) +-spec on_publish_done(message(), publish_done_args(), publish_done_env()) -> ok. +on_publish_done(#message{timestamp = Timestamp}, + #{session_rebirth_time := Created}, + #{ignore_before_create := IgnoreBeforeCreate}) when IgnoreBeforeCreate, Timestamp < Created -> ok; -on_publish_done(#message{timestamp = Timestamp} = Msg, _, _, Threshold, Counter) -> +on_publish_done(#message{timestamp = Timestamp} = Msg, + _, + #{threshold := Threshold, counter := Counter}) -> case ?NOW - Timestamp of Elapsed when Elapsed > Threshold -> case get_log_quota(Counter) of @@ -202,7 +212,7 @@ init_topk_tab(_) -> , {read_concurrency, true} ]). --spec get_log_quota(counter:counter_ref()) -> boolean(). +-spec get_log_quota(counters:counters_ref()) -> boolean(). get_log_quota(Counter) -> case counters:get(Counter, ?QUOTA_IDX) of Quota when Quota > 0 -> @@ -212,7 +222,7 @@ get_log_quota(Counter) -> false end. --spec set_log_quota(proplists:proplist(), counter:counter_ref()) -> ok. +-spec set_log_quota(proplists:proplist(), counters:counters_ref()) -> ok. set_log_quota(Cfg, Counter) -> MaxLogNum = get_value(max_log_num, Cfg), counters:put(Counter, ?QUOTA_IDX, MaxLogNum). @@ -328,12 +338,15 @@ publish(TickTime, Cfg, Notices) -> load(IgnoreBeforeCreate, Threshold, Counter) -> _ = emqx:hook('message.publish_done', - fun ?MODULE:on_publish_done/5, - [IgnoreBeforeCreate, Threshold, Counter]), + fun ?MODULE:on_publish_done/3, + [#{ignore_before_create => IgnoreBeforeCreate, + threshold => Threshold, + counter => Counter} + ]), ok. unload() -> - emqx:unhook('message.publish_done', fun ?MODULE:on_publish_done/5). + emqx:unhook('message.publish_done', fun ?MODULE:on_publish_done/3). -spec get_topic(proplists:proplist()) -> binary(). get_topic(Cfg) -> diff --git a/src/emqx_session.erl b/src/emqx_session.erl index d4b671e71..6122982ae 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -320,7 +320,8 @@ is_awaiting_full(#session{awaiting_rel = AwaitingRel, puback(PacketId, Session = #session{inflight = Inflight, created_at = CreatedAt}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> - emqx:run_hook('message.publish_done', [Msg, CreatedAt]), + emqx:run_hook('message.publish_done', + [Msg, #{session_rebirth_time => CreatedAt}]), Inflight1 = emqx_inflight:delete(PacketId, Inflight), return_with(Msg, dequeue(Session#session{inflight = Inflight1})); {value, {_Pubrel, _Ts}} -> @@ -346,7 +347,8 @@ pubrec(PacketId, Session = #session{inflight = Inflight, created_at = CreatedAt} case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> %% execute hook here, because message record will be replaced by pubrel - emqx:run_hook('message.publish_done', [Msg, CreatedAt]), + emqx:run_hook('message.publish_done', + [Msg, #{session_rebirth_time => CreatedAt}]), Inflight1 = emqx_inflight:update(PacketId, with_ts(pubrel), Inflight), {ok, Msg, Session#session{inflight = Inflight1}}; {value, {pubrel, _Ts}} -> @@ -443,7 +445,8 @@ deliver([Msg | More], Acc, Session) -> end. deliver_msg(Msg = #message{qos = ?QOS_0}, Session) -> - emqx:run_hook('message.publish_done', [Msg, Session#session.created_at]), + emqx:run_hook('message.publish_done', + [Msg, #{session_rebirth_time => Session#session.created_at}]), {ok, [{undefined, maybe_ack(Msg)}], Session}; deliver_msg(Msg = #message{qos = QoS}, Session = From 6bd5fd218ec421b144f7c75211f46e12ca4b1da2 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 15 Nov 2021 11:02:45 +0800 Subject: [PATCH 048/104] chore: limit/page to position/bytes (#6161) --- .../src/emqx_trace/emqx_trace_api.erl | 25 ++++++++++++------- .../test/emqx_mod_trace_api_SUITE.erl | 8 +++--- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index a5e5b2906..3af272776 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -16,6 +16,7 @@ -module(emqx_trace_api). -include_lib("emqx/include/logger.hrl"). +-include_lib("kernel/include/file.hrl"). %% API -export([ list_trace/2 @@ -75,7 +76,7 @@ download_zip_log(#{name := Name}, _Param) -> ZipDir = emqx_trace:zip_dir(), Zips = group_trace_file(ZipDir, TraceLog, TraceFiles), ZipFileName = ZipDir ++ TraceLog, - {ok, ZipFile} = zip:zip(ZipFileName, Zips), + {ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]), emqx_trace:delete_files_after_send(ZipFileName, Zips), {ok, #{}, {sendfile, 0, filelib:file_size(ZipFile), ZipFile}}; {error, Reason} -> @@ -88,7 +89,7 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) -> {ok, Node, Bin} -> ZipName = ZipDir ++ Node ++ "-" ++ TraceLog, ok = file:write_file(ZipName, Bin), - [ZipName | Acc]; + [Node ++ "-" ++ TraceLog | Acc]; {error, Node, Reason} -> ?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]), Acc @@ -101,20 +102,19 @@ collect_trace_file(TraceLog) -> BadNodes =/= [] andalso ?LOG(error, "download log rpc failed on ~p", [BadNodes]), Files. -%% _page as position and _limit as bytes for front-end reusing components stream_log_file(#{name := Name}, Params) -> Node0 = proplists:get_value(<<"node">>, Params, atom_to_binary(node())), - Position0 = proplists:get_value(<<"_page">>, Params, <<"0">>), - Bytes0 = proplists:get_value(<<"_limit">>, Params, <<"500">>), + Position0 = proplists:get_value(<<"position">>, Params, <<"0">>), + Bytes0 = proplists:get_value(<<"bytes">>, Params, <<"1000">>), Node = binary_to_existing_atom(Node0), Position = binary_to_integer(Position0), Bytes = binary_to_integer(Bytes0), case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of {ok, Bin} -> - Meta = #{<<"page">> => Position + byte_size(Bin), <<"limit">> => Bytes}, + Meta = #{<<"position">> => Position + byte_size(Bin), <<"bytes">> => Bytes}, {ok, #{meta => Meta, items => Bin}}; - eof -> - Meta = #{<<"page">> => Position, <<"limit">> => Bytes}, + {eof, Size} -> + Meta = #{<<"position">> => Size, <<"bytes">> => Bytes}, {ok, #{meta => Meta, items => <<"">>}}; {error, Reason} -> logger:log(error, "read_file_failed by ~p", [{Name, Reason, Position, Bytes}]), @@ -134,6 +134,7 @@ read_trace_file(Name, Position, Limit) -> [] -> {error, not_found} end. +-dialyzer({nowarn_function, read_file/3}). read_file(Path, Offset, Bytes) -> {ok, IoDevice} = file:open(Path, [read, raw, binary]), try @@ -141,7 +142,13 @@ read_file(Path, Offset, Bytes) -> 0 -> ok; _ -> file:position(IoDevice, {bof, Offset}) end, - file:read(IoDevice, Bytes) + case file:read(IoDevice, Bytes) of + {ok, Bin} -> {ok, Bin}; + {error, Reason} -> {error, Reason}; + eof -> + #file_info{size = Size} = file:read_file_info(IoDevice), + {eof, Size} + end after file:close(IoDevice) end. diff --git a/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl index 609a2d93c..fc786dbd0 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl @@ -124,14 +124,14 @@ t_stream_log(_Config) -> {ok, FileBin} = file:read_file(File), ct:pal("FileBin: ~p ~s", [byte_size(FileBin), FileBin]), Header = auth_header_(), - {ok, Binary} = request_api(get, api_path("trace/test_stream_log/log?_limit=10"), Header), + {ok, Binary} = request_api(get, api_path("trace/test_stream_log/log?bytes=10"), Header), #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta, <<"items">> := Bin}} = json(Binary), ?assertEqual(10, byte_size(Bin)), - ?assertEqual(#{<<"page">> => 10, <<"limit">> => 10}, Meta), - Path = api_path("trace/test_stream_log/log?_page=20&_limit=10"), + ?assertEqual(#{<<"position">> => 10, <<"bytes">> => 10}, Meta), + Path = api_path("trace/test_stream_log/log?position=20&bytes=10"), {ok, Binary1} = request_api(get, Path, Header), #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta1, <<"items">> := Bin1}} = json(Binary1), - ?assertEqual(#{<<"page">> => 30, <<"limit">> => 10}, Meta1), + ?assertEqual(#{<<"position">> => 30, <<"bytes">> => 10}, Meta1), ?assertEqual(10, byte_size(Bin1)), unload(), ok. From cace53a02b8b9e535be158955029db6c8c19abac Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 12 Nov 2021 17:24:25 +0800 Subject: [PATCH 049/104] build: update docker build --- .github/workflows/build_packages.yaml | 67 ++++---- .github/workflows/run_automate_tests.yaml | 59 +++---- .github/workflows/run_fvt_tests.yaml | 15 +- Makefile | 11 +- build | 17 ++ deploy/docker/Dockerfile | 21 +-- docker.mk | 188 ---------------------- 7 files changed, 104 insertions(+), 274 deletions(-) delete mode 100644 docker.mk diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index efaf14bfc..4e696dab2 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -308,8 +308,8 @@ jobs: env: PROFILE: ${{ matrix.profile}} run: | - if [ -d /tmp/packages/$PROFILE ]; then - cd /tmp/packages/$PROFILE + if [ -d _packages/$PROFILE ]; then + cd _packages/$PROFILE for var in $(ls emqx-* ); do bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" done @@ -319,7 +319,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/') with: name: ${{ matrix.profile }} - path: /tmp/packages/${{ matrix.profile }}/. + path: _packages/${{ matrix.profile }}/. docker: runs-on: ubuntu-20.04 @@ -329,6 +329,8 @@ jobs: strategy: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + otp: + - 23.3.4.8-1 arch: - [amd64, x86_64] - [arm64v8, aarch64] @@ -348,22 +350,42 @@ jobs: path: . - name: unzip source code run: unzip -q source.zip - - name: build emqx docker image - env: - PROFILE: ${{ matrix.profile }} - ARCH: ${{ matrix.arch[0] }} - QEMU_ARCH: ${{ matrix.arch[1] }} - run: | - sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - cd source - sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH make docker - cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd - - - uses: actions/upload-artifact@v1 - if: startsWith(github.ref, 'refs/tags/') + - uses: docker/setup-buildx-action@v1 + - uses: docker/setup-qemu-action@v1 with: - name: ${{ matrix.profile }} - path: source/_packages/${{ matrix.profile }}/. + image: tonistiigi/binfmt:latest + platforms: all + - uses: docker/metadata-action@v3 + id: meta + with: + images: ${{ github.repository_owner }}/${{ matrix.profile }} + flavor: | + latest=${{ !github.event.release.prerelease }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + type=semver,pattern={{version}}-otp${{ matrix.otp }} + type=semver,pattern={{major}}.{{minor}}-otp${{ matrix.otp }} + - uses: docker/login-action@v1 + if: github.event_name == 'release' + with: + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + - uses: docker/build-push-action@v2 + with: + push: ${{ github.event_name == 'release' }} + pull: true + no-cache: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4:${{ matrix.otp }}-alpine3.14 + RUN_FROM=alpine:3.14 + EMQX_NAME=${{ matrix.profile }} + file: source/deploy/docker/Dockerfile + context: source delete-artifact: runs-on: ubuntu-20.04 @@ -443,15 +465,6 @@ jobs: -X POST \ -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - - name: push docker image to docker hub - if: github.event_name == 'release' - run: | - set -e -x -u - sudo make docker-prepare - cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd - - echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin - sudo TARGET=emqx/${{ matrix.profile }} make docker-push - sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list - name: update repo.emqx.io if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee' run: | diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml index fb213c713..4bea3f128 100644 --- a/.github/workflows/run_automate_tests.yaml +++ b/.github/workflows/run_automate_tests.yaml @@ -10,14 +10,19 @@ on: jobs: build: + strategy: + matrix: + otp: + - 23.3.4.8-1 + runs-on: ubuntu-latest outputs: - imgname: ${{ steps.build_docker.outputs.imgname}} - version: ${{ steps.build_docker.outputs.version}} + imgname: ${{ steps.prepare.outputs.imgname}} + version: ${{ steps.prepare.outputs.version}} steps: - uses: actions/checkout@v2 - - name: build docker - id: build_docker + - name: prepare + id: prepare run: | if [ -f EMQX_ENTERPRISE ]; then echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials @@ -25,18 +30,20 @@ jobs: echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token make deps-emqx-ee make clean - fi - make docker - echo "::set-output name=version::$(./pkg-vsn.sh)" - if [ -f EMQX_ENTERPRISE ]; then echo "::set-output name=imgname::emqx-ee" + echo "::set-output name=version::$(./pkg-vsn.sh)" else echo "::set-output name=imgname::emqx" + echo "::set-output name=version::$(./pkg-vsn.sh)" fi + - name: build docker image + run: | + make ${{ steps.prepare.outputs.imgname }}-docker + docker save emqx/${{ steps.prepare.outputs.imgname }}:${{ steps.prepare.outputs.version }} -o image.tar.gz - uses: actions/upload-artifact@v2 with: - name: emqx-docker-image-zip - path: _packages/${{ steps.build_docker.outputs.imgname }}/${{ steps.build_docker.outputs.imgname }}-docker-${{ steps.build_docker.outputs.version }}.zip + name: image + path: image.tar.gz webhook: runs-on: ubuntu-latest @@ -52,15 +59,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image-zip + name: image path: /tmp - name: load docker image - env: - imgname: ${{ needs.build.outputs.imgname}} - version: ${{ needs.build.outputs.version }} run: | - unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp - docker load < /tmp/${imgname}-docker-${version} + docker load < /tmp/image.tar.gz - name: docker compose up timeout-minutes: 5 env: @@ -152,15 +155,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image-zip + name: image path: /tmp - name: load docker image - env: - imgname: ${{ needs.build.outputs.imgname }} - version: ${{ needs.build.outputs.version }} run: | - unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp - docker load < /tmp/${imgname}-docker-${version} + docker load < /tmp/image.tar.gz - name: docker compose up timeout-minutes: 5 env: @@ -259,15 +258,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image-zip + name: image path: /tmp - name: load docker image - env: - imgname: ${{ needs.build.outputs.imgname }} - version: ${{ needs.build.outputs.version }} run: | - unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp - docker load < /tmp/${imgname}-docker-${version} + docker load < /tmp/image.tar.gz - name: docker compose up timeout-minutes: 5 env: @@ -355,15 +350,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image-zip + name: image path: /tmp - name: load docker image - env: - imgname: ${{ needs.build.outputs.imgname }} - version: ${{ needs.build.outputs.version }} run: | - unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp - docker load < /tmp/${imgname}-docker-${version} + docker load < /tmp/image.tar.gz - name: docker compose up timeout-minutes: 5 env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 80fced0e4..8eecc9528 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -25,13 +25,15 @@ jobs: echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token make deps-emqx-ee echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV + echo "PROFILE=emqx-ee" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV else echo "TARGET=emqx/emqx" >> $GITHUB_ENV + echo "PROFILE=emqx" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV fi - name: make emqx image - run: make docker + run: make ${PROFILE}-docker - name: run emqx timeout-minutes: 5 run: | @@ -79,11 +81,15 @@ jobs: echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token make deps-emqx-ee echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV + echo "PROFILE=emqx-ee" >> $GITHUB_ENV + echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV else echo "TARGET=emqx/emqx" >> $GITHUB_ENV + echo "PROFILE=emqx" >> $GITHUB_ENV + echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV fi - name: make emqx image - run: make docker + run: make ${PROFILE}-docker - name: install k3s env: KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" @@ -105,11 +111,10 @@ jobs: KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" timeout-minutes: 5 run: | - version=$(./pkg-vsn.sh) - sudo docker save ${TARGET}:$version -o emqx.tar.gz + sudo docker save ${TARGET}:${EMQX_TAG} -o emqx.tar.gz sudo k3s ctr image import emqx.tar.gz - sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/charts/emqx/Chart.yaml + sed -i -r "s/^appVersion: .*$/appVersion: \"${EMQX_TAG}\"/g" deploy/charts/emqx/Chart.yaml sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml helm install emqx \ diff --git a/Makefile b/Makefile index 11edd8929..e758892e9 100644 --- a/Makefile +++ b/Makefile @@ -145,11 +145,18 @@ $1: $1-rel endef $(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt)))) +## docker target is to create docker instructions +.PHONY: $(REL_PROFILES:%=%-docker) +define gen-docker-target +$1-docker: $(COMMON_DEPS) + @$(BUILD) $1 docker +endef +ALL_ZIPS = $(REL_PROFILES) +$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt)))) + .PHONY: run run: $(PROFILE) quickrun .PHONY: quickrun quickrun: ./_build/$(PROFILE)/rel/emqx/bin/emqx console - -include docker.mk diff --git a/build b/build index dbbfcbdfb..7caf2579d 100755 --- a/build +++ b/build @@ -127,6 +127,20 @@ make_zip() { (cd "${tard}" && zip -qr - emqx) > "${zipball}" } +make_docker() { + ## Build Docker image + echo "DOCKER BUILD: Build Docker image." + echo "DOCKER BUILD: build version -> $PKG_VSN." + echo "DOCKER BUILD: docker repo -> emqx/$PROFILE " + + docker build --no-cache \ + --build-arg BUILD_FROM="ghcr.io/emqx/emqx-builder/4.4:${OTP:-23.3.4.8-1}-alpine3.14" \ + --build-arg RUN_FROM="alpine:3.14" \ + --build-arg EMQX_NAME="$PROFILE" \ + --tag "emqx/$PROFILE:$PKG_VSN" \ + -f deploy/docker/Dockerfile . +} + log "building artifact=$ARTIFACT for profile=$PROFILE" case "$ARTIFACT" in @@ -147,6 +161,9 @@ case "$ARTIFACT" in make -C "deploy/packages/${PKGERDIR}" clean EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}" ;; + docker) + make_docker + ;; *) log "Unknown artifact $ARTIFACT" exit 1 diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index e362c6b73..ae11ad424 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,10 +1,7 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine-amd64 -ARG RUN_FROM=alpine:3.12 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4:23.3.4.8-1-alpine3.14 +ARG RUN_FROM=alpine:3.14 FROM ${BUILD_FROM} AS builder -ARG QEMU_ARCH=x86_64 -COPY tmp/qemu-$QEMU_ARCH-stati* /usr/bin/ - RUN apk add --no-cache \ git \ curl \ @@ -32,21 +29,9 @@ RUN cd /emqx \ FROM $RUN_FROM -# Basic build-time metadata as defined at http://label-schema.org -LABEL org.label-schema.docker.dockerfile="Dockerfile" \ - org.label-schema.license="GNU" \ - org.label-schema.name="emqx" \ - org.label-schema.version=${PKG_VSN} \ - org.label-schema.description="EMQ (Erlang MQTT Broker) is a distributed, massively scalable, highly extensible MQTT messaging broker written in Erlang/OTP." \ - org.label-schema.url="https://emqx.io" \ - org.label-schema.vcs-type="Git" \ - org.label-schema.vcs-url="https://github.com/emqx/emqx" \ - maintainer="EMQ X Team " - -ARG QEMU_ARCH=x86_64 ARG EMQX_NAME=emqx -COPY deploy/docker/docker-entrypoint.sh tmp/qemu-$QEMU_ARCH-stati* /usr/bin/ +COPY deploy/docker/docker-entrypoint.sh /usr/bin/ COPY --from=builder /emqx/_build/$EMQX_NAME/rel/emqx /opt/emqx RUN ln -s /opt/emqx/bin/* /usr/local/bin/ diff --git a/docker.mk b/docker.mk deleted file mode 100644 index 22f2b6e4f..000000000 --- a/docker.mk +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/make -f -# -*- makefile -*- - -## default globals. -## when built with `make docker` command the default profile is either emqx or emqx-ee (for enterprise) -## or the TARGET varialbe can be set beforehand to force a different name -TARGET ?= emqx/$(PROFILE) -QEMU_ARCH ?= x86_64 -ARCH ?= amd64 -QEMU_VERSION ?= v5.0.0-2 -OS ?= alpine -export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) - -ifeq ($(findstring emqx-ee, $(TARGET)), emqx-ee) - ARCH_LIST := amd64 arm64v8 arm32v7 - EMQX_NAME := emqx-ee -else ifeq ($(findstring emqx-edge, $(TARGET)), emqx-edge) - ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x - EMQX_NAME := emqx-edge -else - ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x - EMQX_NAME := emqx -endif - -.PHONY: docker -docker: docker-build docker-tag docker-save - -.PHONY: docker-prepare -docker-prepare: - ## Prepare the machine before any code installation scripts - # @echo "PREPARE: Setting up dependencies." - # @apt update -y - # @apt install --only-upgrade docker-ce -y - - ## Update docker configuration to enable docker manifest command - @echo "PREPARE: Updating docker configuration" - @mkdir -p $$HOME/.docker - - # enable experimental to use docker manifest command - @echo '{ "experimental": "enabled" }' | tee $$HOME/.docker/config.json - # enable experimental - @echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50 }' | tee /etc/docker/daemon.json - @service docker restart - -.PHONY: docker-build -docker-build: - ## Build Docker image - @echo "DOCKER BUILD: Build Docker image." - @echo "DOCKER BUILD: build version -> $(PKG_VSN)." - @echo "DOCKER BUILD: arch - $(ARCH)." - @echo "DOCKER BUILD: qemu arch - $(QEMU_ARCH)." - @echo "DOCKER BUILD: docker repo - $(TARGET) " - @echo "DOCKER BUILD: emqx name - $(EMQX_NAME)." - - ## Prepare qemu to build images other then x86_64 on travis - @echo "PREPARE: Qemu" \ - && docker run --rm --privileged multiarch/qemu-user-static:register --reset - - @mkdir -p tmp \ - && cd tmp \ - && curl -L -o qemu-$(QEMU_ARCH)-static.tar.gz https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-$(QEMU_ARCH)-static.tar.gz \ - && tar xzf qemu-$(QEMU_ARCH)-static.tar.gz \ - && cd - - - @docker build --no-cache \ - --build-arg PKG_VSN=$(PKG_VSN) \ - --build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine-$(ARCH) \ - --build-arg RUN_FROM=$(ARCH)/alpine:3.12 \ - --build-arg EMQX_NAME=$(EMQX_NAME) \ - --build-arg QEMU_ARCH=$(QEMU_ARCH) \ - --tag $(TARGET):build-$(OS)-$(ARCH) \ - -f deploy/docker/Dockerfile . - -.PHONY: docker-tag -docker-tag: - @echo "DOCKER TAG: Tag Docker image." - @for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):build-$(OS)-$${arch})" ]; then \ - docker tag $(TARGET):build-$(OS)-$${arch} $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \ - echo "DOCKER TAG: $(TARGET):$(PKG_VSN)-$(OS)-$${arch}"; \ - if [ $${arch} = amd64 ]; then \ - docker tag $(TARGET):$(PKG_VSN)-$(OS)-amd64 $(TARGET):$(PKG_VSN); \ - echo "DOCKER TAG: $(TARGET):$(PKG_VSN)"; \ - fi; \ - fi; \ - done - -.PHONY: docker-save -docker-save: - @echo "DOCKER SAVE: Save Docker image." - - @mkdir -p _packages/$(EMQX_NAME) - - @if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \ - docker save $(TARGET):$(PKG_VSN) > $(EMQX_NAME)-docker-$(PKG_VSN); \ - zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN).zip $(EMQX_NAME)-docker-$(PKG_VSN); \ - mv ./$(EMQX_NAME)-docker-$(PKG_VSN).zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN).zip; \ - fi - - @for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker save $(TARGET):$(PKG_VSN)-$(OS)-$${arch} > $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \ - zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \ - mv ./$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip; \ - fi; \ - done - -.PHONY: docker-push -docker-push: - @echo "DOCKER PUSH: Push Docker image."; - @echo "DOCKER PUSH: pushing - $(TARGET):$(PKG_VSN)."; - - @if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \ - docker push $(TARGET):$(PKG_VSN); \ - docker tag $(TARGET):$(PKG_VSN) $(TARGET):latest; \ - docker push $(TARGET):latest; \ - fi; - - @for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker push $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \ - fi; \ - done - -.PHONY: docker-manifest-list -docker-manifest-list: - version="docker manifest create --amend $(TARGET):$(PKG_VSN)"; \ - latest="docker manifest create --amend $(TARGET):latest"; \ - for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ];then \ - version="$${version} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \ - latest="$${latest} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \ - fi; \ - done; \ - eval $$version; \ - eval $$latest; - - for arch in $(ARCH_LIST); do \ - case $${arch} in \ - "amd64") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \ - fi; \ - ;; \ - "arm64v8") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \ - fi; \ - ;; \ - "arm32v7") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \ - fi; \ - ;; \ - "i386") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \ - fi; \ - ;; \ - "s390x") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \ - fi; \ - ;; \ - esac; \ - done; - - docker manifest inspect $(TARGET):$(PKG_VSN) - docker manifest push $(TARGET):$(PKG_VSN); - docker manifest inspect $(TARGET):latest - docker manifest push $(TARGET):latest; - -.PHONY: docker-clean -docker-clean: - @echo "DOCKER CLEAN: Clean Docker image." - - @if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then docker rmi -f $$(docker images -q $(TARGET):$(PKG_VSN)); fi - - @for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker rmi -f $$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch}); \ - fi \ - done From 23e2bd62c59a65d4f38ae661cb3e166c49839692 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 15 Nov 2021 15:53:49 +0800 Subject: [PATCH 050/104] fix: there should not be multiple layers of directories when download trace zip file (#6165) --- apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl | 4 ++-- apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl | 5 ++--- lib-ce/emqx_modules/src/emqx_mod_trace_api.erl | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index 3af272776..e6c87d69f 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -75,10 +75,10 @@ download_zip_log(#{name := Name}, _Param) -> TraceFiles = collect_trace_file(TraceLog), ZipDir = emqx_trace:zip_dir(), Zips = group_trace_file(ZipDir, TraceLog, TraceFiles), - ZipFileName = ZipDir ++ TraceLog, + ZipFileName = ZipDir ++ binary_to_list(Name) ++ ".zip", {ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]), emqx_trace:delete_files_after_send(ZipFileName, Zips), - {ok, #{}, {sendfile, 0, filelib:file_size(ZipFile), ZipFile}}; + {ok, ZipFile}; {error, Reason} -> {error, Reason} end. diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index 1bd28d888..169fd50bc 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -336,9 +336,8 @@ t_download_log(_Config) -> {ok, _} = emqtt:connect(Client), [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], ct:sleep(100), - {ok, #{}, {sendfile, 0, ZipFileSize, _ZipFile}} = - emqx_trace_api:download_zip_log(#{name => Name}, []), - ?assert(ZipFileSize > 0), + {ok, ZipFile} = emqx_trace_api:download_zip_log(#{name => Name}, []), + ?assert(filelib:file_size(ZipFile) > 0), ok = emqtt:disconnect(Client), unload(), ok. diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl index a3abbedf7..99fb97796 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl @@ -87,7 +87,7 @@ update_trace(Path, Params) -> download_zip_log(Path, Params) -> case emqx_trace_api:download_zip_log(Path, Params) of - {ok, _Header, _File}= Return -> Return; + {ok, File} -> minirest:return_file(File); {error, _Reason} = Err -> return(Err) end. From 61c68ddb35a269821ae30d2daf259583691d7766 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Mon, 15 Nov 2021 10:27:08 +0100 Subject: [PATCH 051/104] feat(rpc): Bump gen_rpc version --- priv/emqx.schema | 24 ++++++++++++++++++++++++ rebar.config | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/priv/emqx.schema b/priv/emqx.schema index 602de56b9..b4f008a6c 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -362,11 +362,35 @@ end}. ]}. %% RPC server port. +{mapping, "rpc.driver", "gen_rpc.driver", +[ {default, tcp} +, {datatype, {enum, [tcp, ssl]}} +]}. + {mapping, "rpc.tcp_server_port", "gen_rpc.tcp_server_port", [ {default, 5369}, {datatype, integer} ]}. +%% RPC SSL server port. +{mapping, "rpc.enable_ssl", "gen_rpc.ssl_server_port", [ + {default, 5369}, + {datatype, integer} +]}. + +%% RPC SSL certificates +{mapping, "rpc.certfile", "gen_rpc.certfile", [ + {datatype, string} +]}. + +{mapping, "rpc.keyfile", "gen_rpc.keyfile", [ + {datatype, string} +]}. + +{mapping, "rpc.cacertfile", "gen_rpc.cacertfile", [ + {datatype, string} +]}. + %% Number of tcp connections when connecting to RPC server {mapping, "rpc.tcp_client_num", "gen_rpc.tcp_client_num", [ {default, 0}, diff --git a/rebar.config b/rebar.config index cc147dec3..52f07a12a 100644 --- a/rebar.config +++ b/rebar.config @@ -44,7 +44,7 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.2"}}} - , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.6.0"}}} + , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.7.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} From 96d2615cc84137368902b0ea09db21e71c0cb2b5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 15 Nov 2021 17:41:22 +0800 Subject: [PATCH 052/104] fix: return error code when trace log not foundd (#6168) --- lib-ce/emqx_modules/src/emqx_mod_trace_api.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl index 99fb97796..1daab1520 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl @@ -88,8 +88,11 @@ update_trace(Path, Params) -> download_zip_log(Path, Params) -> case emqx_trace_api:download_zip_log(Path, Params) of {ok, File} -> minirest:return_file(File); - {error, _Reason} = Err -> return(Err) + {error, Reason} -> return({error, 'NOT_FOUND', Reason}) end. stream_log_file(Path, Params) -> - return(emqx_trace_api:stream_log_file(Path, Params)). + case emqx_trace_api:stream_log_file(Path, Params) of + {ok, File} -> return({ok, File}); + {error, Reason} -> return({error, 'NOT_FOUND', Reason}) + end. From fce93c5a17ec0773a85fd99f3eed5293fb396ce7 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Mon, 15 Nov 2021 17:44:23 +0800 Subject: [PATCH 053/104] feat(migration): improve modules migration and add test cases --- .../src/emqx_mgmt_data_backup.erl | 54 ++++++++++++--- ...emqx_mongo_auth_module_migration_SUITE.erl | 65 +++++++++++++++++++ .../e4.2.8.json | 1 + .../e4.3.5.json | 1 + 4 files changed, 112 insertions(+), 9 deletions(-) create mode 100644 apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl create mode 100644 apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json create mode 100644 apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 6e467a8ba..4aa66b0fc 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -237,10 +237,12 @@ import_resource(#{<<"id">> := Id, config => Config, created_at => NCreatedAt, description => Desc}). + import_resources_and_rules(Resources, Rules, FromVersion) when FromVersion =:= "4.0" orelse FromVersion =:= "4.1" orelse - FromVersion =:= "4.2" -> + FromVersion =:= "4.2" orelse + FromVersion =:= "4.3" -> Configs = lists:foldl(fun compatible_version/2 , [], Resources), lists:foreach(fun(#{<<"actions">> := Actions} = Rule) -> NActions = apply_new_config(Actions, Configs), @@ -305,6 +307,17 @@ compatible_version(#{<<"id">> := ID, {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}), NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)), [{ID, #{headers => NHeaders, method => Method}} | Acc]; + +compatible_version(#{<<"id">> := ID, + <<"type">> := Type, + <<"config">> := Config} = Resource, Acc) + when Type =:= <<"backend_mongo_single">> + orelse Type =:= <<"backend_mongo_sharded">> + orelse Type =:= <<"backend_mongo_rs">> -> + NewConfig = maps:merge(#{<<"srv_record">> => false}, Config), + {ok, _Resource} = import_resource(Resource#{<<"config">> := NewConfig}), + [{ID, NewConfig} | Acc]; + % normal version compatible_version(Resource, Acc) -> {ok, _Resource} = import_resource(Resource), @@ -511,16 +524,39 @@ import_modules(Modules) -> undefined -> ok; _ -> - lists:foreach(fun(#{<<"id">> := Id, - <<"type">> := Type, - <<"config">> := Config, - <<"enabled">> := Enabled, - <<"created_at">> := CreatedAt, - <<"description">> := Description}) -> - _ = emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description}) - end, Modules) + NModules = migrate_modules(Modules), + lists:foreach(fun(#{<<"id">> := Id, + <<"type">> := Type, + <<"config">> := Config, + <<"enabled">> := Enabled, + <<"created_at">> := CreatedAt, + <<"description">> := Description}) -> + _ = emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description}) + end, NModules) end. +migrate_modules(Modules) -> + migrate_modules(Modules, []). + +migrate_modules([], Acc) -> + lists:reverse(Acc); +migrate_modules([#{<<"type">> := <<"mongo_authentication">>, + <<"config">> := Config} = Module | More], Acc) -> + WMode = case maps:get(<<"w_mode">>, Config, <<"unsafe">>) of + <<"undef">> -> <<"unsafe">>; + Other -> Other + end, + RMode = case maps:get(<<"r_mode">>, Config, <<"master">>) of + <<"undef">> -> <<"master">>; + <<"slave-ok">> -> <<"slave_ok">>; + Other0 -> Other0 + end, + NConfig = Config#{<<"srv_record">> => false, + <<"w_mode">> => WMode, + <<"r_mode">> => RMode}, + migrate_modules(More, [Module#{<<"config">> => NConfig} | Acc]); +migrate_modules([Module | More], Acc) -> + migrate_modules(More, [Module | Acc]). import_schemas(Schemas) -> case ets:info(emqx_schema) of diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl new file mode 100644 index 000000000..3333c3087 --- /dev/null +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mongo_auth_module_migration_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx_modules/include/emqx_modules.hrl"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx_modules_spec), + emqx_ct_helpers:start_apps([emqx_management, emqx_modules]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_modules, emqx_management]), + application:unload(emqx_modules_spec), + ok. + +-ifdef(EMQX_ENTERPRISE). + +t_import_4_2(Config) -> + ?assertMatch(ok, import("e4.2.8.json", Config)), + timer:sleep(100), + + MongoAuthNModule = emqx_modules_registry:find_module_by_type(mongo_authentication), + ?assertNotEqual(not_found, MongoAuthNModule), + ?assertMatch(#module{config = #{<<"srv_record">> := _}}, MongoAuthNModule), + delete_modules(). + +t_import_4_3(Config) -> + ?assertMatch(ok, import("e4.3.5.json", Config)), + timer:sleep(100), + + MongoAuthNModule = emqx_modules_registry:find_module_by_type(mongo_authentication), + ?assertNotEqual(not_found, MongoAuthNModule), + ?assertMatch(#module{config = #{<<"srv_record">> := _}}, MongoAuthNModule), + delete_modules(). + +-endif. + +import(File, Config) -> + Filename = filename:join(proplists:get_value(data_dir, Config), File), + emqx_mgmt_data_backup:import(Filename, "{}"). + +delete_modules() -> + [emqx_modules_registry:remove_module(Mod) || Mod <- emqx_modules_registry:get_modules()]. diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json new file mode 100644 index 000000000..0ea956a93 --- /dev/null +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json @@ -0,0 +1 @@ +{"version":"4.2","date":"2021-11-15 01:52:40","modules":[{"id":"module:79002e0f","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:34834081","type":"presence","config":{"qos":0},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:f6eb69d1","type":"recon","config":{},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:7ae737b2","type":"mongo_authentication","config":{"w_mode":"undef","verify":false,"type":"single","super_query_selector":"","super_query_field":"","super_query_collection":"","ssl":false,"server":"127.0.0.1:27017","r_mode":"undef","pool_size":8,"password":"public","login":"admin","keyfile":{"filename":"","file":""},"database":"mqtt","certfile":{"filename":"","file":""},"cacertfile":{"filename":"","file":""},"auth_source":"admin","auth_query_selector":"username=%u","auth_query_password_hash":"sha256","auth_query_password_field":"password","auth_query_collection":"mqtt_user","acl_query_selectors":[],"acl_query_collection":"mqtt_acl"},"enabled":false,"created_at":1636941148794,"description":""},{"id":"module:e8c63201","type":"internal_acl","config":{"acl_rule_file":"etc/acl.conf"},"enabled":true,"created_at":1636941076704,"description":""}],"rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"qP5m2iS9qnn51gHoGLbaiMo/GwE=","tags":"administrator"}],"auth_mnesia":[],"acl_mnesia":[],"schemas":[],"configs":[],"listeners_state":[]} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json new file mode 100644 index 000000000..48828a9c0 --- /dev/null +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json @@ -0,0 +1 @@ +{"version":"4.3","rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"/mWV4UgV0xmVUZX4qdIXQvxXZB0=","tags":"administrator"}],"auth_mnesia":[],"acl_mnesia":[],"modules":[{"id":"module:5881add2","type":"mongo_authentication","config":{"w_mode":"undef","verify":false,"type":"single","super_query_selector":"","super_query_field":"","super_query_collection":"","ssl":false,"server":"127.0.0.1:27017","r_mode":"undef","pool_size":8,"password":"public","login":"admin","keyfile":{"filename":"","file":""},"database":"mqtt","certfile":{"filename":"","file":""},"cacertfile":{"filename":"","file":""},"auth_source":"admin","auth_query_selector":"username=%u","auth_query_password_hash":"sha256","auth_query_password_field":"password","auth_query_collection":"mqtt_user","acl_query_selectors":[],"acl_query_collection":"mqtt_acl"},"enabled":false,"created_at":1636942609573,"description":""},{"id":"module:2adb6480","type":"presence","config":{"qos":0},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:24fabe8a","type":"internal_acl","config":{"acl_rule_file":"etc/acl.conf"},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:22c70ab8","type":"recon","config":{},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:a59f9a4a","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1636942586725,"description":""}],"schemas":[],"configs":[],"listeners_state":[],"date":"2021-11-15 10:16:56"} \ No newline at end of file From 2f279b36075739b3ee0c07f6c8dfa9a523e7dc1b Mon Sep 17 00:00:00 2001 From: zhouzb Date: Mon, 15 Nov 2021 18:14:51 +0800 Subject: [PATCH 054/104] test(migration): fix code scope --- .../test/emqx_mongo_auth_module_migration_SUITE.erl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl index 3333c3087..3a697a17d 100644 --- a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl @@ -20,11 +20,16 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). + +-ifdef(EMQX_ENTERPRISE). -include_lib("emqx_modules/include/emqx_modules.hrl"). +-endif. all() -> emqx_ct:all(?MODULE). +-ifdef(EMQX_ENTERPRISE). + init_per_suite(Config) -> application:load(emqx_modules_spec), emqx_ct_helpers:start_apps([emqx_management, emqx_modules]), @@ -35,8 +40,6 @@ end_per_suite(_Config) -> application:unload(emqx_modules_spec), ok. --ifdef(EMQX_ENTERPRISE). - t_import_4_2(Config) -> ?assertMatch(ok, import("e4.2.8.json", Config)), timer:sleep(100), @@ -55,11 +58,11 @@ t_import_4_3(Config) -> ?assertMatch(#module{config = #{<<"srv_record">> := _}}, MongoAuthNModule), delete_modules(). --endif. - import(File, Config) -> Filename = filename:join(proplists:get_value(data_dir, Config), File), emqx_mgmt_data_backup:import(Filename, "{}"). delete_modules() -> [emqx_modules_registry:remove_module(Mod) || Mod <- emqx_modules_registry:get_modules()]. + +-endif. From ca1458d4d7e1040d6fcc7597d5d68b36a316e652 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 15 Nov 2021 15:34:48 +0100 Subject: [PATCH 055/104] chore(emqx_rule_engine): bump app vsn to 4.4.0 --- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 66af0da21..98e5487e2 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.6"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, From a6032d54352d975d1e169bd55c9384faf98c2d35 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 15 Nov 2021 23:02:28 +0100 Subject: [PATCH 056/104] build: update default otp versions --- .tool-versions | 2 +- build | 6 ++++-- deploy/docker/Dockerfile | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.tool-versions b/.tool-versions index 200caa93b..a42915120 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -erlang 23.2.7.2-emqx-2 +erlang 23.3.4.9-2 diff --git a/build b/build index 7caf2579d..5cd022dcc 100755 --- a/build +++ b/build @@ -14,6 +14,8 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" export OTP_VSN +EMQX_RUNNER_IMAGE='alpine:3.14' +EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE}}" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" export PKG_VSN @@ -134,8 +136,8 @@ make_docker() { echo "DOCKER BUILD: docker repo -> emqx/$PROFILE " docker build --no-cache \ - --build-arg BUILD_FROM="ghcr.io/emqx/emqx-builder/4.4:${OTP:-23.3.4.8-1}-alpine3.14" \ - --build-arg RUN_FROM="alpine:3.14" \ + --build-arg BUILD_FROM="${EMQX_BUILDER}" \ + --build-arg RUN_FROM="{EMQX_RUNNER_IMAGE}" \ --build-arg EMQX_NAME="$PROFILE" \ --tag "emqx/$PROFILE:$PKG_VSN" \ -f deploy/docker/Dockerfile . diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index ae11ad424..b48011bfd 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4:23.3.4.8-1-alpine3.14 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-alpine3.14 ARG RUN_FROM=alpine:3.14 FROM ${BUILD_FROM} AS builder From f711f78c5c591dec92adee63a8cdd672b5c52903 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 15 Nov 2021 23:12:31 +0100 Subject: [PATCH 057/104] build: pin otp version 23.3.4.9-2 --- .github/workflows/build_packages.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 4e696dab2..85d14bbfb 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -11,7 +11,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} @@ -134,7 +134,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} erl_otp: - - 23.2.7.2-emqx-2 + - 23.3.4.9-2 exclude: - profile: emqx-edge @@ -213,7 +213,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 23.3.4.8-1 + - 23.3.4.9-2 arch: - amd64 - arm64 @@ -300,7 +300,7 @@ jobs: -v $(pwd):/emqx \ --workdir /emqx \ --platform linux/$ARCH \ - ghcr.io/emqx/emqx-builder/4.4:$OTP-$SYSTEM \ + ghcr.io/emqx/emqx-builder/4.4-1:$OTP-$SYSTEM \ bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \ make $PROFILE-pkg || cat rebar3.crashdump; \ EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" @@ -330,7 +330,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 23.3.4.8-1 + - 23.3.4.9-2 arch: - [amd64, x86_64] - [arm64v8, aarch64] @@ -381,7 +381,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4:${{ matrix.otp }}-alpine3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile From d350281270cae5fe9423edeaa9c3f7a6b2fbdd42 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 15 Nov 2021 23:33:23 +0100 Subject: [PATCH 058/104] build: pin ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 --- .ci/build_packages/Dockerfile | 2 +- .ci/docker-compose-file/docker-compose.yaml | 2 +- .github/workflows/build_slim_packages.yaml | 6 +++--- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 4 ++-- .github/workflows/run_test_cases.yaml | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile index 6f9a12159..0a7c7496d 100644 --- a/.ci/build_packages/Dockerfile +++ b/.ci/build_packages/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 FROM ${BUILD_FROM} ARG EMQX_NAME=emqx diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index a502aeb6e..0415ac142 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 env_file: - conf.env environment: diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index bf85578c5..5ee2a63dd 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -15,12 +15,12 @@ jobs: strategy: matrix: erl_otp: - - erl23.2.7.2-emqx-2 + - 23.3.4.9-2 os: - ubuntu20.04 - centos7 - container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-1:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 @@ -58,7 +58,7 @@ jobs: strategy: matrix: erl_otp: - - 23.2.7.2-emqx-2 + - 23.3.4.9-2 steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 0aa8f7903..5ebd2f69d 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 855d9463c..4cd2a914b 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 8eecc9528..4bc0d13b2 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -185,7 +185,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -234,7 +234,7 @@ jobs: relup_test_build: needs: relup_test_plan runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 defaults: run: shell: bash diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 3fbe88c40..9d002138e 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_static_analysis: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 steps: - uses: actions/checkout@v2 @@ -27,7 +27,7 @@ jobs: run_proper_test: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 steps: - uses: actions/checkout@v2 From b68f01e7e718b4f4db66f6cad2d795f75a9b90d7 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 15 Nov 2021 23:42:19 +0100 Subject: [PATCH 059/104] build: do not print 'otp' prefix for otp version --- scripts/get-otp-vsn.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/get-otp-vsn.sh b/scripts/get-otp-vsn.sh index fd5b4fd0b..a791318dc 100755 --- a/scripts/get-otp-vsn.sh +++ b/scripts/get-otp-vsn.sh @@ -2,6 +2,4 @@ set -euo pipefail -vsn=$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell) - -echo "otp${vsn}" +erl -noshell -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' From c97c6aefc9dfb718a707a2322c82ac579dcd3e86 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 00:46:58 +0100 Subject: [PATCH 060/104] build: rename zip package and rpm deb packages --- .ci/fvt_tests/local_relup_test_run.sh | 4 ++++ .ci/fvt_tests/relup.lux | 8 +++++--- .github/workflows/build_packages.yaml | 11 ++++++----- .github/workflows/run_fvt_tests.yaml | 2 ++ build | 9 ++++----- deploy/packages/deb/Makefile | 2 +- deploy/packages/deb/debian/control | 2 +- deploy/packages/rpm/Makefile | 9 +++++---- deploy/packages/rpm/emqx.spec | 2 +- 9 files changed, 29 insertions(+), 20 deletions(-) diff --git a/.ci/fvt_tests/local_relup_test_run.sh b/.ci/fvt_tests/local_relup_test_run.sh index 7ef40b0f2..576ed6d02 100755 --- a/.ci/fvt_tests/local_relup_test_run.sh +++ b/.ci/fvt_tests/local_relup_test_run.sh @@ -15,6 +15,8 @@ PROFILE="$1" VSN="$2" OLD_VSN="$3" PACKAGE_PATH="$4" +FROM_OTP_VSN="${5:-23.3.4.9-2}" +TO_OTP_VSN="${6:-23.3.4.9-2}" TEMPDIR=$(mktemp -d) trap '{ rm -rf -- "$TEMPDIR"; }' EXIT @@ -37,4 +39,6 @@ exec docker run \ --var ONE_MORE_EMQX_PATH="/relup_test/one_more_emqx" \ --var VSN="$VSN" \ --var OLD_VSN="$OLD_VSN" \ + --var FROM_OTP_VSN="$FROM_OTP_VSN" \ + --var TO_OTP_VSN="$TO_OTP_VSN" \ relup.lux diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index 2940f5ce0..6da46a706 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -3,6 +3,8 @@ [config var=ONE_MORE_EMQX_PATH] [config var=VSN] [config var=OLD_VSN] +[config var=FROM_OTP_VSN] +[config var=TO_OTP_VSN] [config shell_cmd=/bin/bash] [config timeout=600000] @@ -19,7 +21,7 @@ [shell emqx] !cd $PACKAGE_PATH - !unzip -q -o $PROFILE-ubuntu20.04-$(echo $OLD_VSN | sed -r 's/[v|e]//g')-amd64.zip + !unzip -q -o $PROFILE-$(echo $OLD_VSN | sed -r 's/[v|e]//g')-otp${FROM_OTP_VSN}-ubuntu20.04-amd64.zip ?SH-PROMPT !cd emqx @@ -80,7 +82,7 @@ !echo "" > log/emqx.log.1 ?SH-PROMPT - !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ + !cp -f ../$PROFILE-$VSN-otp${TO_OTP_VSN}-ubuntu20.04-amd64.zip releases/ !./bin/emqx install $VSN ?Made release permanent: "$VSN" @@ -105,7 +107,7 @@ !echo "" > log/emqx.log.1 ?SH-PROMPT - !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ + !cp -f ../$PROFILE-$VSN-otp${TO_OTP_VSN}-ubuntu20.04-amd64.zip releases/ !./bin/emqx install $VSN ?Made release permanent: "$VSN" diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 85d14bbfb..f468c8c04 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -263,7 +263,7 @@ jobs: run: unzip -q source.zip - name: downloads old emqx zip packages env: - OTP_VSN: otp${{ matrix.otp }} + OTP_VSN: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} @@ -282,10 +282,11 @@ jobs: cd source/_upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for tag in ${old_vsns[@]}; do - if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip) | grep -oE "^[23]+")" ];then - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip.sha256 - echo "$(cat ${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip.sha256) ${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip" | sha256sum -c || exit 1 + package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" + if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256 + echo "$(cat $package_name.zip.sha256) $package_name.zip" | sha256sum -c || exit 1 fi done - name: build emqx packages diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 4bc0d13b2..82fc2b64c 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -321,6 +321,8 @@ jobs: --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ --var VSN="$VSN" \ --var OLD_VSN="$OLD_VSN" \ + --var FROM_OTP_VSN="23.3.4.9-2" \ + --var TO_OTP_VSN="23.3.4.9-2" \ emqx_built/.ci/fvt_tests/relup.lux - uses: actions/upload-artifact@v2 name: Save debug data diff --git a/build b/build index 5cd022dcc..50a2df1de 100755 --- a/build +++ b/build @@ -12,10 +12,9 @@ ARTIFACT="$2" # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" -OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" -export OTP_VSN EMQX_RUNNER_IMAGE='alpine:3.14' -EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE}}" +EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')" +EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" export PKG_VSN @@ -81,7 +80,7 @@ make_relup() { rm -rf "$tmp_dir" fi releases+=( "$base_vsn" ) - done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-$SYSTEM*-$ARCH.zip" -type f) + done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f) fi if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" @@ -121,7 +120,7 @@ make_zip() { log "ERROR: $tarball is not found" fi local zipball - zipball="${pkgpath}/${PROFILE}-${OTP_VSN}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip" + zipball="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" tar zxf "${tarball}" -C "${tard}/emqx" ## try to be portable for zip packages. ## for DEB and RPM packages the dependencies are resoved by yum and apt diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 6f3956409..1f709d860 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -8,7 +8,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(OTP_VSN)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) .PHONY: all all: | $(BUILT) diff --git a/deploy/packages/deb/debian/control b/deploy/packages/deb/debian/control index e35535c12..b3794036c 100644 --- a/deploy/packages/deb/debian/control +++ b/deploy/packages/deb/debian/control @@ -4,7 +4,7 @@ Priority: optional Maintainer: emqx Build-Depends: debhelper (>=9) Standards-Version: 3.9.6 -Homepage: https://www.emqx.io +Homepage: https://www.emqx.com Package: emqx Architecture: any diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index e8c01e935..0e6cf9c5d 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -6,7 +6,7 @@ dash := - none := space := $(none) $(none) RPM_VSN ?= $(shell echo $(PKG_VSN) | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?") -RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]") +RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]+") ARCH ?= amd64 ifeq ($(ARCH),mips64) @@ -16,12 +16,12 @@ endif EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz -TARGET_PKG := $(EMQX_NAME)-$(OTP_VSN)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) ifeq ($(RPM_REL),) # no tail RPM_REL := 1 endif -SOURCE_PKG := emqx-$(SYSTEM)-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) +SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL)-otp$(OTP_VSN)-$(SYSTEM)-$(shell uname -m) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C @@ -47,7 +47,8 @@ all: | $(BUILT) --define "_service_dst $(SERVICE_DST)" \ --define "_post_addition $(POST_ADDITION)" \ --define "_preun_addition $(PREUN_ADDITION)" \ - --define "_ostype -$(SYSTEM)" \ + --define "_ostype $(SYSTEM)" \ + --define "_otp_vsn $(OTP_VSN)" \ --define "_sharedstatedir /var/lib" \ emqx.spec mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec index 882b7753e..0c309a8d0 100644 --- a/deploy/packages/rpm/emqx.spec +++ b/deploy/packages/rpm/emqx.spec @@ -5,7 +5,7 @@ %define _log_dir %{_var}/log/%{_name} %define _lib_home /usr/lib/%{_name} %define _var_home %{_sharedstatedir}/%{_name} -%define _build_name_fmt %{_arch}/%{_name}%{?_ostype}-%{_version}-%{_release}.%{_arch}.rpm +%define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}-otp%{_otp_vsn}-%{?_ostype}.%{_arch}.rpm %define _build_id_links none Name: %{_package_name} From 3f1fc64a98acfdccf05c17c1f16b67ff40480c22 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 01:10:52 +0100 Subject: [PATCH 061/104] build: fix source rpm package name --- deploy/packages/rpm/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 0e6cf9c5d..5ad80cd37 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -21,7 +21,7 @@ ifeq ($(RPM_REL),) # no tail RPM_REL := 1 endif -SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL)-otp$(OTP_VSN)-$(SYSTEM)-$(shell uname -m) +SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL)-otp$(OTP_VSN)-$(SYSTEM).$(shell uname -m) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C From 755dd11b070cf8f0d3909cc3d0cafc942cdb1df5 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 01:28:42 +0100 Subject: [PATCH 062/104] build: always pull image before build --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index 50a2df1de..08580de70 100755 --- a/build +++ b/build @@ -134,7 +134,7 @@ make_docker() { echo "DOCKER BUILD: build version -> $PKG_VSN." echo "DOCKER BUILD: docker repo -> emqx/$PROFILE " - docker build --no-cache \ + docker build --no-cache --pull \ --build-arg BUILD_FROM="${EMQX_BUILDER}" \ --build-arg RUN_FROM="{EMQX_RUNNER_IMAGE}" \ --build-arg EMQX_NAME="$PROFILE" \ From a178b6cc083617d24cdf5f50836a1af90b0a6e89 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 16 Nov 2021 11:27:05 +0800 Subject: [PATCH 063/104] ci: fix make docker error for github action --- .github/workflows/run_automate_tests.yaml | 7 ++----- .github/workflows/run_fvt_tests.yaml | 14 ++++++-------- build | 14 ++++++++------ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml index 4bea3f128..e4680be78 100644 --- a/.github/workflows/run_automate_tests.yaml +++ b/.github/workflows/run_automate_tests.yaml @@ -10,11 +10,6 @@ on: jobs: build: - strategy: - matrix: - otp: - - 23.3.4.8-1 - runs-on: ubuntu-latest outputs: imgname: ${{ steps.prepare.outputs.imgname}} @@ -37,6 +32,8 @@ jobs: echo "::set-output name=version::$(./pkg-vsn.sh)" fi - name: build docker image + env: + OTP_VSN: 23.3.4.9-2 run: | make ${{ steps.prepare.outputs.imgname }}-docker docker save emqx/${{ steps.prepare.outputs.imgname }}:${{ steps.prepare.outputs.version }} -o image.tar.gz diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 82fc2b64c..2aece7312 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -13,10 +13,6 @@ jobs: steps: - uses: actions/checkout@v1 - - uses: gleam-lang/setup-erlang@v1.1.2 - id: install_erlang - with: - otp-version: 23.2 - name: prepare run: | if make emqx-ee --dry-run > /dev/null 2>&1; then @@ -24,6 +20,7 @@ jobs: git config --global credential.helper store echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token make deps-emqx-ee + make clean echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV echo "PROFILE=emqx-ee" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV @@ -33,6 +30,8 @@ jobs: echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV fi - name: make emqx image + env: + OTP_VSN: 23.3.4.9-2 run: make ${PROFILE}-docker - name: run emqx timeout-minutes: 5 @@ -69,10 +68,6 @@ jobs: steps: - uses: actions/checkout@v1 - - uses: gleam-lang/setup-erlang@v1.1.2 - id: install_erlang - with: - otp-version: 23.2 - name: prepare run: | if make emqx-ee --dry-run > /dev/null 2>&1; then @@ -80,6 +75,7 @@ jobs: git config --global credential.helper store echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token make deps-emqx-ee + make clean echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV echo "PROFILE=emqx-ee" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV @@ -89,6 +85,8 @@ jobs: echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV fi - name: make emqx image + env: + OTP_VSN: 23.3.4.9-2 run: make ${PROFILE}-docker - name: install k3s env: diff --git a/build b/build index 08580de70..f29dbc985 100755 --- a/build +++ b/build @@ -12,10 +12,6 @@ ARTIFACT="$2" # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" -EMQX_RUNNER_IMAGE='alpine:3.14' -EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')" -EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}" - PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" export PKG_VSN @@ -129,14 +125,20 @@ make_zip() { } make_docker() { + EMQX_RUNNER_IMAGE='alpine:3.14' + EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')" + EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}" + ## Build Docker image echo "DOCKER BUILD: Build Docker image." - echo "DOCKER BUILD: build version -> $PKG_VSN." echo "DOCKER BUILD: docker repo -> emqx/$PROFILE " + echo "DOCKER BUILD: build version -> $PKG_VSN." + echo "DOCKER BUILD: build from -> $EMQX_BUILDER." + echo "DOCKER BUILD: runner from -> $EMQX_RUNNER_IMAGE." docker build --no-cache --pull \ --build-arg BUILD_FROM="${EMQX_BUILDER}" \ - --build-arg RUN_FROM="{EMQX_RUNNER_IMAGE}" \ + --build-arg RUN_FROM="${EMQX_RUNNER_IMAGE}" \ --build-arg EMQX_NAME="$PROFILE" \ --tag "emqx/$PROFILE:$PKG_VSN" \ -f deploy/docker/Dockerfile . From 2dc63cffea4f039c7b1132a437777efdd80ff168 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 08:37:53 +0100 Subject: [PATCH 064/104] fix(emqx_misc): call gen_tcp to decide if ipv6_probe is supported --- src/emqx_misc.erl | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl index 2b77d0455..01495460f 100644 --- a/src/emqx_misc.erl +++ b/src/emqx_misc.erl @@ -66,16 +66,9 @@ maybe_parse_ip(Host) -> %% @doc Add `ipv6_probe' socket option if it's supported. ipv6_probe(Opts) -> - case persistent_term:get({?MODULE, ipv6_probe_supported}, unknown) of - unknown -> - %% e.g. 23.2.7.1-emqx-2-x86_64-unknown-linux-gnu-64 - OtpVsn = emqx_vm:get_otp_version(), - Bool = (match =:= re:run(OtpVsn, "emqx", [{capture, none}])), - _ = persistent_term:put({?MODULE, ipv6_probe_supported}, Bool), - ipv6_probe(Bool, Opts); - Bool -> - ipv6_probe(Bool, Opts) - end. + Bool = try gen_tcp:ipv6_probe() + catch _ : _ -> false end, + ipv6_probe(Bool, Opts). ipv6_probe(false, Opts) -> Opts; ipv6_probe(true, Opts) -> [{ipv6_probe, true} | Opts]. From 2251159c4fe613c10c416a5bd22702da719c0fc4 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 16 Nov 2021 17:51:38 +0800 Subject: [PATCH 065/104] build: pin otp version to 23.3.4.9-3 and builder version to 4.4-2 --- .github/workflows/build_packages.yaml | 12 ++++++------ .github/workflows/build_slim_packages.yaml | 6 +++--- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_automate_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 12 ++++++------ .github/workflows/run_test_cases.yaml | 4 ++-- build | 2 +- deploy/docker/Dockerfile | 2 +- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index f468c8c04..83c10a4c9 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -11,7 +11,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} @@ -134,7 +134,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} erl_otp: - - 23.3.4.9-2 + - 23.3.4.9-3 exclude: - profile: emqx-edge @@ -213,7 +213,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 23.3.4.9-2 + - 23.3.4.9-3 arch: - amd64 - arm64 @@ -301,7 +301,7 @@ jobs: -v $(pwd):/emqx \ --workdir /emqx \ --platform linux/$ARCH \ - ghcr.io/emqx/emqx-builder/4.4-1:$OTP-$SYSTEM \ + ghcr.io/emqx/emqx-builder/4.4-2:$OTP-$SYSTEM \ bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \ make $PROFILE-pkg || cat rebar3.crashdump; \ EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" @@ -331,7 +331,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 23.3.4.9-2 + - 23.3.4.9-3 arch: - [amd64, x86_64] - [arm64v8, aarch64] @@ -382,7 +382,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:${{ matrix.otp }}-alpine3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 5ee2a63dd..5888dff8b 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -15,12 +15,12 @@ jobs: strategy: matrix: erl_otp: - - 23.3.4.9-2 + - 23.3.4.9-3 os: - ubuntu20.04 - centos7 - container: ghcr.io/emqx/emqx-builder/4.4-1:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-2:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 @@ -58,7 +58,7 @@ jobs: strategy: matrix: erl_otp: - - 23.3.4.9-2 + - 23.3.4.9-3 steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 5ebd2f69d..b8c4a5c18 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 4cd2a914b..5f452194c 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml index e4680be78..ca7a84612 100644 --- a/.github/workflows/run_automate_tests.yaml +++ b/.github/workflows/run_automate_tests.yaml @@ -33,7 +33,7 @@ jobs: fi - name: build docker image env: - OTP_VSN: 23.3.4.9-2 + OTP_VSN: 23.3.4.9-3 run: | make ${{ steps.prepare.outputs.imgname }}-docker docker save emqx/${{ steps.prepare.outputs.imgname }}:${{ steps.prepare.outputs.version }} -o image.tar.gz diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 2aece7312..a16586bd9 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -31,7 +31,7 @@ jobs: fi - name: make emqx image env: - OTP_VSN: 23.3.4.9-2 + OTP_VSN: 23.3.4.9-3 run: make ${PROFILE}-docker - name: run emqx timeout-minutes: 5 @@ -86,7 +86,7 @@ jobs: fi - name: make emqx image env: - OTP_VSN: 23.3.4.9-2 + OTP_VSN: 23.3.4.9-3 run: make ${PROFILE}-docker - name: install k3s env: @@ -183,7 +183,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -232,7 +232,7 @@ jobs: relup_test_build: needs: relup_test_plan runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 defaults: run: shell: bash @@ -319,8 +319,8 @@ jobs: --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ --var VSN="$VSN" \ --var OLD_VSN="$OLD_VSN" \ - --var FROM_OTP_VSN="23.3.4.9-2" \ - --var TO_OTP_VSN="23.3.4.9-2" \ + --var FROM_OTP_VSN="23.3.4.9-3" \ + --var TO_OTP_VSN="23.3.4.9-3" \ emqx_built/.ci/fvt_tests/relup.lux - uses: actions/upload-artifact@v2 name: Save debug data diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 9d002138e..2f5570e28 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_static_analysis: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 steps: - uses: actions/checkout@v2 @@ -27,7 +27,7 @@ jobs: run_proper_test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/build b/build index f29dbc985..24e79dda3 100755 --- a/build +++ b/build @@ -127,7 +127,7 @@ make_zip() { make_docker() { EMQX_RUNNER_IMAGE='alpine:3.14' EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')" - EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}" + EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-2:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}" ## Build Docker image echo "DOCKER BUILD: Build Docker image." diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index b48011bfd..88f80bdf1 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-alpine3.14 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14 ARG RUN_FROM=alpine:3.14 FROM ${BUILD_FROM} AS builder From 39c564c072d7934c29d543dc7a72c510a80ded71 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 16 Nov 2021 17:55:27 +0800 Subject: [PATCH 066/104] ci: rename windows packae --- .github/workflows/build_packages.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 83c10a4c9..46133e95b 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -65,6 +65,8 @@ jobs: strategy: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + otp: + - 23.2 exclude: - profile: emqx-edge @@ -79,7 +81,7 @@ jobs: - uses: gleam-lang/setup-erlang@v1.1.0 id: install_erlang with: - otp-version: 23.2 + otp-version: ${{ matrix.otp }} - name: build env: PYTHON: python @@ -90,10 +92,10 @@ jobs: $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" - $pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip" + $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip" } else { - $pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip" + $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip" } cd source ## We do not build/release bcrypt for windows package From 284d1223728e43f796bdcfb6c04cd076f4993029 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 16 Nov 2021 18:33:21 +0800 Subject: [PATCH 067/104] ci: pin otp version to 23.3.4.9-3 and builder version to 4.4-2 --- .ci/build_packages/Dockerfile | 2 +- .ci/docker-compose-file/docker-compose.yaml | 2 +- .ci/fvt_tests/local_relup_test_run.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile index 0a7c7496d..1063a2c13 100644 --- a/.ci/build_packages/Dockerfile +++ b/.ci/build_packages/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 FROM ${BUILD_FROM} ARG EMQX_NAME=emqx diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 0415ac142..8bd6455e9 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.ci/fvt_tests/local_relup_test_run.sh b/.ci/fvt_tests/local_relup_test_run.sh index 576ed6d02..8a563a217 100755 --- a/.ci/fvt_tests/local_relup_test_run.sh +++ b/.ci/fvt_tests/local_relup_test_run.sh @@ -15,8 +15,8 @@ PROFILE="$1" VSN="$2" OLD_VSN="$3" PACKAGE_PATH="$4" -FROM_OTP_VSN="${5:-23.3.4.9-2}" -TO_OTP_VSN="${6:-23.3.4.9-2}" +FROM_OTP_VSN="${5:-23.3.4.9-3}" +TO_OTP_VSN="${6:-23.3.4.9-3}" TEMPDIR=$(mktemp -d) trap '{ rm -rf -- "$TEMPDIR"; }' EXIT From db802ad04ff347d5fafc3fe3691fd6094449b549 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 11:35:13 +0100 Subject: [PATCH 068/104] chore: update toos-versions to pin 23.3.4.9-3 --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index a42915120..757309f18 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -erlang 23.3.4.9-2 +erlang 23.3.4.9-3 From 4d9854012e959800ed69384d772f0b35fdab7ef1 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 14:08:58 +0100 Subject: [PATCH 069/104] build: parameterise path to Dockerfile --- Makefile | 1 + build | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e758892e9..4b15adfc7 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X export EMQX_CE_DASHBOARD_VERSION ?= v4.3.3 +export DOCKERFILE=deploy/docker/Dockerfile ifeq ($(OS),Windows_NT) export REBAR_COLOR=none endif diff --git a/build b/build index 24e79dda3..9375dfc8c 100755 --- a/build +++ b/build @@ -141,7 +141,7 @@ make_docker() { --build-arg RUN_FROM="${EMQX_RUNNER_IMAGE}" \ --build-arg EMQX_NAME="$PROFILE" \ --tag "emqx/$PROFILE:$PKG_VSN" \ - -f deploy/docker/Dockerfile . + -f "${DOCKERFILE}" . } log "building artifact=$ARTIFACT for profile=$PROFILE" From ced24290110c0f7fe7e145582afe8187aa5fc54d Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 14:36:36 +0100 Subject: [PATCH 070/104] fix: bump new feature lib-ce apps to 4.4 --- lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- scripts/apps-version-check.sh | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index e78f1c3f6..1604198dc 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.7"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index 47a3d8888..87c1c083f 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.3.4"}, + {vsn, "4.4.0"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 7c2ea5eb2..241d0fc15 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -2,6 +2,7 @@ set -euo pipefail latest_release=$(git describe --abbrev=0 --tags) +echo "Compare base: $latest_release" bad_app_count=0 From a070708e8d8b2ebed55d8785653e06fcc70f6200 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 18:59:23 +0100 Subject: [PATCH 071/104] build: Move otp version number to RPM's release part --- deploy/packages/rpm/Makefile | 13 ++++--------- deploy/packages/rpm/emqx.spec | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 5ad80cd37..acee8b51c 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -5,8 +5,9 @@ BUILT := $(SRCDIR)/BUILT dash := - none := space := $(none) $(none) -RPM_VSN ?= $(shell echo $(PKG_VSN) | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?") -RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]+") +## RPM does not allow '-' in version nubmer and release string, replace with '_' +RPM_VSN := $(subst -,_,$(PKG_VSN)) +RPM_REL := otp$(subst -,_,$(OTP_VSN)) ARCH ?= amd64 ifeq ($(ARCH),mips64) @@ -17,11 +18,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) -ifeq ($(RPM_REL),) - # no tail - RPM_REL := 1 -endif -SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL)-otp$(OTP_VSN)-$(SYSTEM).$(shell uname -m) +SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C @@ -47,8 +44,6 @@ all: | $(BUILT) --define "_service_dst $(SERVICE_DST)" \ --define "_post_addition $(POST_ADDITION)" \ --define "_preun_addition $(PREUN_ADDITION)" \ - --define "_ostype $(SYSTEM)" \ - --define "_otp_vsn $(OTP_VSN)" \ --define "_sharedstatedir /var/lib" \ emqx.spec mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec index 0c309a8d0..ef63b936c 100644 --- a/deploy/packages/rpm/emqx.spec +++ b/deploy/packages/rpm/emqx.spec @@ -5,7 +5,7 @@ %define _log_dir %{_var}/log/%{_name} %define _lib_home /usr/lib/%{_name} %define _var_home %{_sharedstatedir}/%{_name} -%define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}-otp%{_otp_vsn}-%{?_ostype}.%{_arch}.rpm +%define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}.%{_arch}.rpm %define _build_id_links none Name: %{_package_name} From ca1ece3db0d4ba33fef5fb9fd5bbb87cbd70d2ae Mon Sep 17 00:00:00 2001 From: Turtle Date: Wed, 17 Nov 2021 11:17:10 +0800 Subject: [PATCH 072/104] feat(publish-api): Publish api supports user-properties parameters --- .../src/emqx_mgmt_api_pubsub.erl | 54 ++++++++++++------- .../test/emqx_mgmt_api_SUITE.erl | 24 +++++++-- 2 files changed, 55 insertions(+), 23 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl index 53ca022bb..1d89f237a 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl @@ -71,8 +71,8 @@ subscribe(_Bindings, Params) -> publish(_Bindings, Params) -> logger:debug("API publish Params:~p", [Params]), - {ClientId, Topic, Qos, Retain, Payload} = parse_publish_params(Params), - case do_publish(ClientId, Topic, Qos, Retain, Payload) of + {ClientId, Topic, Qos, Retain, Payload, UserProps} = parse_publish_params(Params), + case do_publish(ClientId, Topic, Qos, Retain, Payload, UserProps) of {ok, MsgIds} -> case proplists:get_value(<<"return">>, Params, undefined) of undefined -> minirest:return(ok); @@ -114,7 +114,8 @@ loop_subscribe([Params | ParamsN], Acc) -> {_, Code0, _Reason} -> Code0 end, Result = #{clientid => ClientId, - topic => resp_topic(proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), + topic => resp_topic(proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), code => Code}, loop_subscribe(ParamsN, [Result | Acc]). @@ -123,12 +124,13 @@ loop_publish(Params) -> loop_publish([], Result) -> lists:reverse(Result); loop_publish([Params | ParamsN], Acc) -> - {ClientId, Topic, Qos, Retain, Payload} = parse_publish_params(Params), - Code = case do_publish(ClientId, Topic, Qos, Retain, Payload) of + {ClientId, Topic, Qos, Retain, Payload, UserProps} = parse_publish_params(Params), + Code = case do_publish(ClientId, Topic, Qos, Retain, Payload, UserProps) of {ok, _} -> 0; {_, Code0, _} -> Code0 end, - Result = #{topic => resp_topic(proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), + Result = #{topic => resp_topic(proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), code => Code}, loop_publish(ParamsN, [Result | Acc]). @@ -143,7 +145,8 @@ loop_unsubscribe([Params | ParamsN], Acc) -> {_, Code0, _} -> Code0 end, Result = #{clientid => ClientId, - topic => resp_topic(proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), + topic => resp_topic(proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), code => Code}, loop_unsubscribe(ParamsN, [Result | Acc]). @@ -158,14 +161,17 @@ do_subscribe(ClientId, Topics, QoS) -> _ -> ok end. -do_publish(ClientId, _Topics, _Qos, _Retain, _Payload) when not (is_binary(ClientId) or (ClientId =:= undefined)) -> +do_publish(ClientId, _Topics, _Qos, _Retain, _Payload, _UserProps) + when not (is_binary(ClientId) or (ClientId =:= undefined)) -> {ok, ?ERROR8, <<"bad clientid: must be string">>}; -do_publish(_ClientId, [], _Qos, _Retain, _Payload) -> +do_publish(_ClientId, [], _Qos, _Retain, _Payload, _UserProps) -> {ok, ?ERROR15, bad_topic}; -do_publish(ClientId, Topics, Qos, Retain, Payload) -> +do_publish(ClientId, Topics, Qos, Retain, Payload, UserProps) -> MsgIds = lists:map(fun(Topic) -> Msg = emqx_message:make(ClientId, Qos, Topic, Payload), - _ = emqx_mgmt:publish(Msg#message{flags = #{retain => Retain}}), + UserProps1 = #{'User-Property' => UserProps}, + _ = emqx_mgmt:publish(Msg#message{flags = #{retain => Retain}, + headers = #{properties => UserProps1}}), emqx_guid:to_hexstr(Msg#message.id) end, Topics), {ok, MsgIds}. @@ -185,19 +191,22 @@ do_unsubscribe(ClientId, Topic) -> parse_subscribe_params(Params) -> ClientId = proplists:get_value(<<"clientid">>, Params), - Topics = topics(filter, proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), + Topics = topics(filter, proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), QoS = proplists:get_value(<<"qos">>, Params, 0), {ClientId, Topics, QoS}. parse_publish_params(Params) -> - Topics = topics(name, proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), - ClientId = proplists:get_value(<<"clientid">>, Params), - Payload = decode_payload(proplists:get_value(<<"payload">>, Params, <<>>), - proplists:get_value(<<"encoding">>, Params, <<"plain">>)), - Qos = proplists:get_value(<<"qos">>, Params, 0), - Retain = proplists:get_value(<<"retain">>, Params, false), - Payload1 = maybe_maps_to_binary(Payload), - {ClientId, Topics, Qos, Retain, Payload1}. + Topics = topics(name, proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), + ClientId = proplists:get_value(<<"clientid">>, Params), + Payload = decode_payload(proplists:get_value(<<"payload">>, Params, <<>>), + proplists:get_value(<<"encoding">>, Params, <<"plain">>)), + Qos = proplists:get_value(<<"qos">>, Params, 0), + Retain = proplists:get_value(<<"retain">>, Params, false), + Payload1 = maybe_maps_to_binary(Payload), + UserProps = check_user_props(proplists:get_value(<<"user_properties">>, Params, [])), + {ClientId, Topics, Qos, Retain, Payload1, UserProps}. parse_unsubscribe_params(Params) -> ClientId = proplists:get_value(<<"clientid">>, Params), @@ -251,3 +260,8 @@ maybe_maps_to_binary(Payload) -> _C : _E : S -> error({encode_payload_fail, S}) end. + +check_user_props(UserProps) when is_list(UserProps) -> + UserProps; +check_user_props(UserProps) -> + error({user_properties_type_error, UserProps}). diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index f3bc42f3a..a2d3aef0b 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -401,7 +401,7 @@ t_pubsub(_) -> ClientId = <<"client1">>, Options = #{clientid => ClientId, - proto_ver => 5}, + proto_ver => v5}, Topic = <<"mytopic">>, {ok, C1} = emqtt:start_link(Options), {ok, _} = emqtt:connect(C1), @@ -536,11 +536,29 @@ t_pubsub(_) -> api_path(["mqtt/unsubscribe_batch"]), [], auth_header_(), Body3), loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data3), [return_maps]))), + {ok, _, [1]} = emqtt:subscribe(C1, <<"mytopic">>, qos1), + timer:sleep(50), + + %% user properties + {ok, Code} = request_api(post, api_path(["mqtt/publish"]), [], auth_header_(), + #{<<"clientid">> => ClientId, + <<"topic">> => <<"mytopic">>, + <<"qos">> => 1, + <<"payload">> => <<"hello world">>, + <<"user_properties">> => #{<<"porp_1">> => <<"porp_1">>}}), + ?assert(receive + {publish, #{payload := <<"hello world">>, + properties := #{'User-Property' := [{<<"porp_1">>,<<"porp_1">>}]}}} -> + true + after 100 -> + false + end), + ok = emqtt:disconnect(C1), - ?assertEqual(3, emqx_metrics:val('messages.qos1.received') - Qos1Received), + ?assertEqual(4, emqx_metrics:val('messages.qos1.received') - Qos1Received), ?assertEqual(2, emqx_metrics:val('messages.qos2.received') - Qos2Received), - ?assertEqual(5, emqx_metrics:val('messages.received') - Received). + ?assertEqual(6, emqx_metrics:val('messages.received') - Received). loop([]) -> []; From 45965a3e7108a94048a4b410fdd7c26deef04a00 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 17 Nov 2021 10:08:38 +0800 Subject: [PATCH 073/104] ci: update emqx-ci-helper version --- .github/workflows/build_packages.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 46133e95b..ef152604f 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -475,7 +475,7 @@ jobs: -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ - -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \ + -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" - name: update repo.emqx.io if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' @@ -484,7 +484,7 @@ jobs: -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ - -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \ + -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" - name: update homebrew packages if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' @@ -494,7 +494,7 @@ jobs: -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ - -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \ + -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches" fi - uses: geekyeggo/delete-artifact@v1 From af5f93d81a1b5bf8f9d9aad3e20f1319177fafdf Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 17 Nov 2021 10:09:28 +0800 Subject: [PATCH 074/104] build: show macos version --- scripts/get-distro.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh index ae52abba3..00e95e1d8 100755 --- a/scripts/get-distro.sh +++ b/scripts/get-distro.sh @@ -6,7 +6,9 @@ set -euo pipefail if [ "$(uname -s)" = 'Darwin' ]; then - echo 'macos' + DIST='macos' + VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g') + SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" elif [ "$(uname -s)" = 'Linux' ]; then if grep -q -i 'centos' /etc/*-release; then DIST='centos' @@ -15,5 +17,6 @@ elif [ "$(uname -s)" = 'Linux' ]; then DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" fi - echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g' + SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')" fi +echo "$SYSTEM" From 834240a76078205025a75047081b63f05bd2873b Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 17 Nov 2021 15:20:03 +0800 Subject: [PATCH 075/104] ci: cancel otp vsn for docker image tag add otp vsn for docker labels --- .github/workflows/build_packages.yaml | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index ef152604f..73600e91f 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -334,17 +334,6 @@ jobs: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - 23.3.4.9-3 - arch: - - [amd64, x86_64] - - [arm64v8, aarch64] - - [arm32v7, arm] - - [i386, i386] - - [s390x, s390x] - exclude: - - profile: emqx-ee - arch: [i386, i386] - - profile: emqx-ee - arch: [s390x, s390x] steps: - uses: actions/download-artifact@v2 @@ -368,8 +357,10 @@ jobs: type=ref,event=branch type=ref,event=pr type=ref,event=tag - type=semver,pattern={{version}}-otp${{ matrix.otp }} - type=semver,pattern={{major}}.{{minor}}-otp${{ matrix.otp }} + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + labels: + org.opencontainers.image.otp.version=${{ matrix.otp }} - uses: docker/login-action@v1 if: github.event_name == 'release' with: From 35164951e21ebbb053df798fbdb4438017c71d23 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 23:18:30 +0100 Subject: [PATCH 076/104] feat(docker): add flexible docker build The defulat docker build (e.g. make emqx-docker) is based on alpine image and it builds EMQ X from source code. This is not flexible enough when we want to quickly run some tests in a docker container. The new docker build (e.g. make emqx-docker-testing) by default takes the built zip package, and extract it in a very primitive base image such as ubuntu:20.04 and centos:8 --- Makefile | 15 +++++++- build | 62 ++++++++++++++++++++++++++------ deploy/docker/Dockerfile.testing | 43 ++++++++++++++++++++++ 3 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 deploy/docker/Dockerfile.testing diff --git a/Makefile b/Makefile index 4b15adfc7..1f8f17bbd 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,8 @@ export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X export EMQX_CE_DASHBOARD_VERSION ?= v4.3.3 -export DOCKERFILE=deploy/docker/Dockerfile +export DOCKERFILE := deploy/docker/Dockerfile +export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing ifeq ($(OS),Windows_NT) export REBAR_COLOR=none endif @@ -155,6 +156,18 @@ endef ALL_ZIPS = $(REL_PROFILES) $(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt)))) +## emqx-docker-testing +## emqx-ee-docker-testing +## is to directly copy a unzipped zip-package to a +## base image such as ubuntu20.04. Mostly for testing +.PHONY: $(REL_PROFILES:%=%-docker-testing) +define gen-docker-target-testing +$1-docker-testing: $(COMMON_DEPS) + @$(BUILD) $1 docker-testing +endef +ALL_ZIPS = $(REL_PROFILES) +$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target-testing,$(zt)))) + .PHONY: run run: $(PROFILE) quickrun diff --git a/build b/build index 9375dfc8c..50997f76c 100755 --- a/build +++ b/build @@ -124,26 +124,63 @@ make_zip() { (cd "${tard}" && zip -qr - emqx) > "${zipball}" } +## This function builds the default docker image based on alpine:3.14 (by default) make_docker() { - EMQX_RUNNER_IMAGE='alpine:3.14' - EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')" + EMQX_RUNNER_IMAGE="${EMQX_RUNNER_IMAGE:-alpine:3.14}" + EMQX_RUNNER_IMAGE_COMPACT="$(echo "$EMQX_RUNNER_IMAGE" | tr -d ':')" EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-2:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}" - - ## Build Docker image - echo "DOCKER BUILD: Build Docker image." - echo "DOCKER BUILD: docker repo -> emqx/$PROFILE " - echo "DOCKER BUILD: build version -> $PKG_VSN." - echo "DOCKER BUILD: build from -> $EMQX_BUILDER." - echo "DOCKER BUILD: runner from -> $EMQX_RUNNER_IMAGE." - + set -x docker build --no-cache --pull \ --build-arg BUILD_FROM="${EMQX_BUILDER}" \ --build-arg RUN_FROM="${EMQX_RUNNER_IMAGE}" \ --build-arg EMQX_NAME="$PROFILE" \ - --tag "emqx/$PROFILE:$PKG_VSN" \ + --tag "emqx/$PROFILE:${PKG_VSN}" \ -f "${DOCKERFILE}" . } +## This function accepts any base docker image, +## a emqx zip-image, and a image tag (for the image to be built), +## to build a docker image which runs EMQ X +## +## Export below variables to quickly build an image +## +## Name Default Example +## --------------------------------------------------------------------- +## EMQX_BASE_IMAGE current os centos:7 +## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.zip +## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag +## +make_docker_testing() { + if [ -z "${EMQX_BASE_IMAGE:-}" ]; then + case "$SYSTEM" in + ubuntu20*) + EMQX_BASE_IMAGE="ubuntu:20.04" + ;; + centos8) + EMQX_BASE_IMAGE="centos:8" + ;; + *) + echo "Unsupported testing base image for $SYSTEM" + exit 1 + ;; + esac + fi + EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}" + local defaultzip + defaultzip="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" + local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}" + if [ ! -f "$zip" ]; then + log "ERROR: $zip not built?" + exit 1 + fi + set -x + docker build \ + --build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \ + --build-arg EMQX_ZIP_PACKAGE="${zip}" \ + --tag "$EMQX_IMAGE_TAG" \ + -f "${DOCKERFILE_TESTING}" . +} + log "building artifact=$ARTIFACT for profile=$PROFILE" case "$ARTIFACT" in @@ -167,6 +204,9 @@ case "$ARTIFACT" in docker) make_docker ;; + docker-testing) + make_docker_testing + ;; *) log "Unknown artifact $ARTIFACT" exit 1 diff --git a/deploy/docker/Dockerfile.testing b/deploy/docker/Dockerfile.testing new file mode 100644 index 000000000..02490272d --- /dev/null +++ b/deploy/docker/Dockerfile.testing @@ -0,0 +1,43 @@ +ARG BUILD_FROM +FROM ${BUILD_FROM} + +## all we need is the unzip command +RUN if command -v yum; then yum update -y && yum install -y unzip; fi +RUN if command -v apt-get; then apt-get update -y && apt-get install unzip; fi + +ARG EMQX_ZIP_PACKAGE +COPY ${EMQX_ZIP_PACKAGE} /opt/emqx.zip +RUN unzip -q /opt/emqx.zip -d /opt/ && rm /opt/emqx.zip + +COPY deploy/docker/docker-entrypoint.sh /usr/bin/ +RUN ln -s /opt/emqx/bin/* /usr/local/bin/ + +WORKDIR /opt/emqx + +RUN adduser -u 1000 emqx +RUN echo "emqx ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers + +RUN chgrp -Rf emqx /opt/emqx && chmod -Rf g+w /opt/emqx \ + && chown -Rf emqx /opt/emqx + +USER emqx + +VOLUME ["/opt/emqx/log", "/opt/emqx/data", "/opt/emqx/etc"] + +# emqx will occupy these port: +# - 1883 port for MQTT +# - 8081 for mgmt API +# - 8083 for WebSocket/HTTP +# - 8084 for WSS/HTTPS +# - 8883 port for MQTT(SSL) +# - 11883 port for internal MQTT/TCP +# - 18083 for dashboard +# - 4369 epmd (Erlang-distrbution port mapper daemon) listener (deprecated) +# - 4370 default Erlang distrbution port +# - 5369 for gen_rpc port mapping +# - 6369 6370 for distributed node +EXPOSE 1883 8081 8083 8084 8883 11883 18083 4369 4370 5369 6369 6370 + +ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] + +CMD ["/opt/emqx/bin/emqx", "foreground"] From 2a55a712d167b518b52fec9b047c627fc2baccf8 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 17 Nov 2021 13:58:09 +0100 Subject: [PATCH 077/104] build: prepare for 4.4-alpha.1 release --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 4b7da70d2..58cc11811 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.0-alpha.1"}). +-define(EMQX_RELEASE, {opensource, "4.4-alpha.1"}). -else. From eb0f4a543dfb970a114a752106228d7dbca7e463 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 18 Nov 2021 10:35:03 +0800 Subject: [PATCH 078/104] ci: fix upload artifact error --- .github/workflows/build_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 73600e91f..a06d7de08 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -322,7 +322,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/') with: name: ${{ matrix.profile }} - path: _packages/${{ matrix.profile }}/. + path: source/_packages/${{ matrix.profile }}/. docker: runs-on: ubuntu-20.04 From 093a93a7ecd5b472cd54f60f8ea521606b5a4e2c Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 18 Nov 2021 14:27:37 +0800 Subject: [PATCH 079/104] ci: fix not found package when check sha256 --- .github/workflows/build_packages.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index a06d7de08..6f4aab000 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -308,6 +308,7 @@ jobs: make $PROFILE-pkg || cat rebar3.crashdump; \ EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" - name: create sha256 + working-directory: source env: PROFILE: ${{ matrix.profile}} run: | @@ -358,7 +359,6 @@ jobs: type=ref,event=pr type=ref,event=tag type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} labels: org.opencontainers.image.otp.version=${{ matrix.otp }} - uses: docker/login-action@v1 @@ -368,7 +368,7 @@ jobs: password: ${{ secrets.DOCKER_HUB_TOKEN }} - uses: docker/build-push-action@v2 with: - push: ${{ github.event_name == 'release' }} + push: ${{ github.event_name == 'release' && !github.event.release.prerelease }} pull: true no-cache: true platforms: linux/amd64,linux/arm64 From 42333882c8b378a45a8d7048f65775124a621a42 Mon Sep 17 00:00:00 2001 From: lafirest Date: Thu, 18 Nov 2021 14:41:59 +0800 Subject: [PATCH 080/104] fix(emqx_st_statistics): fix unsafe rank range (#6207) * fix(emqx_st_statistics): fix unsafe rank range --- .../emqx_st_statistics_api.erl | 46 +++++---- .../test/emqx_st_statistics_api_SUITE.erl | 98 ++++++++++++------- 2 files changed, 92 insertions(+), 52 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl index ba13d70e2..dc74dcf36 100644 --- a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl @@ -51,25 +51,33 @@ get_history(_Bindings, Params) -> Limit = erlang:binary_to_integer(LimitT), Start = (Page - 1) * Limit + 1, Size = ets:info(?TOPK_TAB, size), - EndT = Start + Limit - 1, - End = erlang:min(EndT, Size), - Infos = lists:foldl(fun(Rank, Acc) -> - [#top_k{topic = Topic - , average_count = Count - , average_elapsed = Elapsed}] = ets:lookup(?TOPK_TAB, Rank), - - Info =[ {rank, Rank} - , {topic, Topic} - , {count, Count} - , {elapsed, Elapsed}], - - [Info | Acc] - end, - [], - lists:seq(Start, End)), - + End = Start + Limit - 1, + {HasNext, Count, Infos} = get_history(Start, End, Size), return({ok, #{meta => #{page => Page, limit => Limit, - hasnext => End < Size, - count => End - Start + 1}, + hasnext => HasNext, + count => Count}, data => Infos}}). + + +get_history(Start, _End, Size) when Start > Size -> + {false, 0, []}; + +get_history(Start, End, Size) when End > Size -> + get_history(Start, Size, Size); + +get_history(Start, End, Size) -> + Fold = fun(Rank, Acc) -> + [#top_k{topic = Topic + , average_count = Count + , average_elapsed = Elapsed}] = ets:lookup(?TOPK_TAB, Rank), + + Info = [ {rank, Rank} + , {topic, Topic} + , {count, Count} + , {elapsed, Elapsed}], + + [Info | Acc] + end, + Infos = lists:foldl(Fold, [], lists:seq(Start, End)), + {End < Size, End - Start + 1, Infos}. diff --git a/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl index 3cf4dafd8..0fb9e904b 100644 --- a/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl @@ -55,23 +55,6 @@ end_per_testcase(_, Config) -> emqx_mod_st_statistics:unload(undefined), Config. -get(Key, ResponseBody) -> - maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])). - -lookup_alarm(Name, [#{<<"name">> := Name} | _More]) -> - true; -lookup_alarm(Name, [_Alarm | More]) -> - lookup_alarm(Name, More); -lookup_alarm(_Name, []) -> - false. - -is_existing(Name, [#{name := Name} | _More]) -> - true; -is_existing(Name, [_Alarm | More]) -> - is_existing(Name, More); -is_existing(_Name, []) -> - false. - t_get_history(_) -> ets:insert(?TOPK_TAB, #top_k{rank = 1, topic = <<"test">>, @@ -81,19 +64,54 @@ t_get_history(_) -> {ok, Data} = request_api(get, api_path(["slow_topic"]), "_page=1&_limit=10", auth_header_()), - Return = #{meta => #{page => 1, - limit => 10, - hasnext => false, - count => 1}, - data => [#{topic => <<"test">>, - rank => 1, - elapsed => 1500, - count => 12}], - code => 0}, + ShouldRet = #{meta => #{page => 1, + limit => 10, + hasnext => false, + count => 1}, + data => [#{topic => <<"test">>, + rank => 1, + elapsed => 1500, + count => 12}], + code => 0}, - ShouldBe = emqx_json:encode(Return), + Ret = decode(Data), - ?assertEqual(ShouldBe, erlang:list_to_binary(Data)). + ?assertEqual(ShouldRet, Ret). + +t_rank_range(_) -> + Insert = fun(Rank) -> + ets:insert(?TOPK_TAB, + #top_k{rank = Rank, + topic = <<"test">>, + average_count = 12, + average_elapsed = 1500}) + end, + lists:foreach(Insert, lists:seq(1, 15)), + + timer:sleep(100), + + {ok, Data} = request_api(get, api_path(["slow_topic"]), "_page=1&_limit=10", + auth_header_()), + + Meta1 = #{page => 1, limit => 10, hasnext => true, count => 10}, + Ret1 = decode(Data), + ?assertEqual(Meta1, maps:get(meta, Ret1)), + + %% End > Size + {ok, Data2} = request_api(get, api_path(["slow_topic"]), "_page=2&_limit=10", + auth_header_()), + + Meta2 = #{page => 2, limit => 10, hasnext => false, count => 5}, + Ret2 = decode(Data2), + ?assertEqual(Meta2, maps:get(meta, Ret2)), + + %% Start > Size + {ok, Data3} = request_api(get, api_path(["slow_topic"]), "_page=3&_limit=10", + auth_header_()), + + Meta3 = #{page => 3, limit => 10, hasnext => false, count => 0}, + Ret3 = decode(Data3), + ?assertEqual(Meta3, maps:get(meta, Ret3)). t_clear(_) -> ets:insert(?TOPK_TAB, #top_k{rank = 1, @@ -106,6 +124,25 @@ t_clear(_) -> ?assertEqual(0, ets:info(?TOPK_TAB, size)). +decode(Data) -> + Pairs = emqx_json:decode(Data), + to_maps(Pairs). + +to_maps([H | _] = List) when is_tuple(H) -> + to_maps(List, #{}); + +to_maps([_ | _] = List) -> + [to_maps(X) || X <- List]; + +to_maps(V) -> V. + +to_maps([{K, V} | T], Map) -> + AtomKey = erlang:binary_to_atom(K), + to_maps(T, Map#{AtomKey => to_maps(V)}); + +to_maps([], Map) -> + Map. + request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). @@ -148,8 +185,3 @@ auth_header_(User, Pass) -> api_path(Parts)-> ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts). - -filter(List, Key, Value) -> - lists:filter(fun(Item) -> - maps:get(Key, Item) == Value - end, List). From d1cf526f34d7c160fbca2b6498501a55ae55c7f1 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 18 Nov 2021 14:43:55 +0800 Subject: [PATCH 081/104] ci: fix permission error when check sha256 --- .github/workflows/build_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 6f4aab000..e1f3e8988 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -315,7 +315,7 @@ jobs: if [ -d _packages/$PROFILE ]; then cd _packages/$PROFILE for var in $(ls emqx-* ); do - bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" + sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" done cd - fi From daeac6edf465aa0d561d0c35af9b92926e551f37 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 18 Nov 2021 15:26:36 +0800 Subject: [PATCH 082/104] chore(release): update emqx release version --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 58cc11811..3e96f1aa7 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4-alpha.1"}). +-define(EMQX_RELEASE, {opensource, "4.4-alpha.2"}). -else. From 099e2a8752aa9f317cda9a4b022f86dc1b131a3f Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 19 Nov 2021 21:54:04 +0100 Subject: [PATCH 083/104] ci: port changes made in master branch --- .github/workflows/build_packages.yaml | 63 +++++++++++++--------- .github/workflows/build_slim_packages.yaml | 21 ++++++-- 2 files changed, 54 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index e1f3e8988..b2104e618 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -1,5 +1,9 @@ name: Cross build packages +concurrency: + group: build-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + on: schedule: - cron: '0 */6 * * *' @@ -11,11 +15,12 @@ on: jobs: prepare: runs-on: ubuntu-20.04 + # prepare source with any OTP version, no need for a matrix container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 outputs: - profiles: ${{ steps.set_profile.outputs.profiles}} - old_vsns: ${{ steps.set_profile.outputs.old_vsns}} + profiles: ${{ steps.set_profile.outputs.profiles }} + old_vsns: ${{ steps.set_profile.outputs.old_vsns }} steps: - uses: actions/checkout@v2 @@ -25,8 +30,8 @@ jobs: - name: set profile id: set_profile shell: bash + working-directory: source run: | - cd source vsn="$(./pkg-vsn.sh)" pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" if make emqx-ee --dry-run > /dev/null 2>&1; then @@ -43,7 +48,7 @@ jobs: run: | make -C source deps-all zip -ryq source.zip source/* source/.[^.]* - - name: get_all_deps + - name: get_all_deps_ee if: endsWith(github.repository, 'enterprise') run: | echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials @@ -63,6 +68,7 @@ jobs: if: endsWith(github.repository, 'emqx') strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: @@ -78,14 +84,16 @@ jobs: - name: unzip source code run: Expand-Archive -Path source.zip -DestinationPath ./ - uses: ilammy/msvc-dev-cmd@v1 - - uses: gleam-lang/setup-erlang@v1.1.0 + - uses: gleam-lang/setup-erlang@v1.1.2 id: install_erlang + ## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows with: otp-version: ${{ matrix.otp }} - name: build env: PYTHON: python DIAGNOSTIC: 1 + working-directory: source run: | $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" @@ -97,7 +105,6 @@ jobs: else { $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip" } - cd source ## We do not build/release bcrypt for windows package Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ if (Test-Path rebar.lock) { @@ -114,8 +121,8 @@ jobs: Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}' > _packages/${{ matrix.profile }}/$pkg_name.sha256 - name: run emqx timeout-minutes: 1 + working-directory: source run: | - cd source ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start Start-Sleep -s 5 ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop @@ -128,18 +135,19 @@ jobs: path: source/_packages/${{ matrix.profile }}/. mac: - runs-on: macos-10.15 - needs: prepare - strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} erl_otp: - 23.3.4.9-3 + macos: + - macos-11 + - macos-10.15 exclude: - profile: emqx-edge - + runs-on: ${{ matrix.macos }} steps: - uses: actions/download-artifact@v2 with: @@ -157,7 +165,7 @@ jobs: id: cache with: path: ~/.kerl - key: erl${{ matrix.erl_otp }}-macos10.15 + key: erl${{ matrix.erl_otp }}-${{ matrix.macos }} - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 @@ -169,18 +177,19 @@ jobs: kerl build ${{ matrix.erl_otp }} kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} - name: build + working-directory: source run: | . $HOME/.kerl/${{ matrix.erl_otp }}/activate - cd source make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 rm -rf _build/${{ matrix.profile }}/lib make ${{ matrix.profile }}-zip - name: test + working-directory: source run: | - cd source - pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip) - unzip -q _packages/${{ matrix.profile }}/$pkg_name + set -x + pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip) + unzip -q $pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' @@ -199,11 +208,11 @@ jobs: ./emqx/bin/emqx_ctl status ./emqx/bin/emqx stop rm -rf emqx - openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}' > ./_packages/${{ matrix.profile }}/$pkg_name.sha256 + openssl dgst -sha256 $pkg_name | awk '{print $2}' > $pkg_name.sha256 - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: - name: ${{ matrix.profile }} + name: ${{ matrix.profile }}-${{ matrix.otp }} path: source/_packages/${{ matrix.profile }}/. linux: @@ -212,6 +221,7 @@ jobs: needs: prepare strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: @@ -270,6 +280,7 @@ jobs: ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }} + working-directory: source run: | set -e -x -u broker=$PROFILE @@ -280,8 +291,8 @@ jobs: export ARCH="arm" fi - mkdir -p source/_upgrade_base - cd source/_upgrade_base + mkdir -p _upgrade_base + cd _upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for tag in ${old_vsns[@]}; do package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" @@ -322,15 +333,15 @@ jobs: - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: - name: ${{ matrix.profile }} + name: ${{ matrix.profile }}-${{ matrix.otp }} path: source/_packages/${{ matrix.profile }}/. docker: runs-on: ubuntu-20.04 - needs: prepare strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: @@ -399,6 +410,8 @@ jobs: strategy: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + otp: + - 23.3.4.9-3 steps: - uses: actions/checkout@v2 @@ -409,7 +422,7 @@ jobs: echo 'EOF' >> $GITHUB_ENV - uses: actions/download-artifact@v2 with: - name: ${{ matrix.profile }} + name: ${{ matrix.profile }}-${{ matrix.otp }} path: ./_packages/${{ matrix.profile }} - name: install dos2unix run: sudo apt-get update && sudo apt install -y dos2unix @@ -460,7 +473,7 @@ jobs: -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - name: update repo.emqx.io - if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee' + if: github.event_name == 'release' && matrix.profile == 'emqx-ee' run: | curl --silent --show-error \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ @@ -469,7 +482,7 @@ jobs: -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" - name: update repo.emqx.io - if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' + if: github.event_name == 'release' && matrix.profile == 'emqx' run: | curl --silent --show-error \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 5888dff8b..23d0e3e53 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -1,5 +1,10 @@ name: Build slim packages +concurrency: + group: slim-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + + on: push: tags: @@ -13,6 +18,7 @@ jobs: runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: erl_otp: - 23.3.4.9-3 @@ -43,7 +49,7 @@ jobs: with: name: rebar3.crashdump path: ./rebar3.crashdump - - name: pakcages test + - name: packages test run: | export CODE_PATH=$GITHUB_WORKSPACE .ci/build_packages/tests.sh @@ -53,12 +59,17 @@ jobs: path: _packages/**/*.zip mac: - runs-on: macos-10.15 strategy: + fail-fast: false matrix: erl_otp: - 23.3.4.9-3 + macos: + - macos-11 + - macos-10.15 + + runs-on: ${{ matrix.macos }} steps: - uses: actions/checkout@v1 @@ -82,7 +93,7 @@ jobs: id: cache with: path: ~/.kerl - key: erl${{ matrix.erl_otp }}-macos10.15 + key: otp-${{ matrix.erl_otp }}-${{ matrix.macos }} - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 @@ -106,8 +117,8 @@ jobs: path: ./rebar3.crashdump - name: test run: | - pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) - unzip -q _packages/${EMQX_NAME}/$pkg_name + pkg_name=$(find _packages/${EMQX_NAME} -mindepth 1 -maxdepth 1 -iname \*.zip) + unzip -q $pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' From 4767b41eb7f0b4efc9af7f81c1cbb3a95c0195df Mon Sep 17 00:00:00 2001 From: lafirest Date: Sun, 21 Nov 2021 18:55:43 +0800 Subject: [PATCH 084/104] fix(emqx_st_statistics): fix initial value error (#6224) * fix(emqx_st_statistics): fix initial value error --- .../src/emqx_st_statistics/emqx_st_statistics.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl index f22aec41c..668cb3926 100644 --- a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl +++ b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl @@ -56,8 +56,8 @@ }. -record(slow_log, { topic :: emqx_types:topic() - , count :: pos_integer() - , elapsed :: pos_integer() + , count :: integer() %% 0 will be used in initial value + , elapsed :: integer() }). -type message() :: #message{}. @@ -233,8 +233,8 @@ update_log(#message{topic = Topic}, Elapsed) -> Topic, [{#slow_log.count, 1}, {#slow_log.elapsed, Elapsed}], #slow_log{topic = Topic, - count = 1, - elapsed = Elapsed}), + count = 0, + elapsed = 0}), ok. -spec do_notification(state()) -> true. From 62dc72c85949b0ec472719b09b9d414288362037 Mon Sep 17 00:00:00 2001 From: Turtle Date: Mon, 22 Nov 2021 11:04:21 +0800 Subject: [PATCH 085/104] feat(sql_rule): test rule add User-Property information --- apps/emqx_rule_engine/src/emqx_rule_events.erl | 3 ++- .../src/emqx_rule_sqltester.erl | 17 ++--------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 97e40439d..72f8345bf 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -454,7 +454,8 @@ columns_with_exam('message.publish') -> , {<<"topic">>, <<"t/a">>} , {<<"qos">>, 1} , {<<"flags">>, #{}} - , {<<"headers">>, undefined} + , {<<"headers">>, #{<<"properties">> => #{<<"User-Property">> => + #{'prop_key' => <<"prop_val">>}}}} , {<<"publish_received_at">>, erlang:system_time(millisecond)} , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index 760205d62..747de87d7 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -98,21 +98,8 @@ sql_test_action() -> fill_default_values(Event, Context) -> maps:merge(envs_examp(Event), Context). -envs_examp(<<"$events/", _/binary>> = EVENT_TOPIC) -> +envs_examp(EVENT_TOPIC) -> EventName = emqx_rule_events:event_name(EVENT_TOPIC), emqx_rule_maps:atom_key_map( maps:from_list( - emqx_rule_events:columns_with_exam(EventName))); -envs_examp(_) -> - #{id => emqx_guid:to_hexstr(emqx_guid:gen()), - clientid => <<"c_emqx">>, - username => <<"u_emqx">>, - payload => <<"{\"id\": 1, \"name\": \"ha\"}">>, - peerhost => <<"127.0.0.1">>, - topic => <<"t/a">>, - qos => 1, - flags => #{sys => true, event => true}, - publish_received_at => emqx_rule_utils:now_ms(), - timestamp => emqx_rule_utils:now_ms(), - node => node() - }. + emqx_rule_events:columns_with_exam(EventName))). From d76275d17d75cf278d412343e698fbfd8701aa87 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 22 Nov 2021 14:59:11 +0800 Subject: [PATCH 086/104] feat: add support ip_address trace options --- apps/emqx_management/src/emqx_mgmt_cli.erl | 142 ++++++++-- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 113 ++++---- .../src/emqx_trace/emqx_trace.erl | 196 ++++++------- .../src/emqx_trace/emqx_trace_api.erl | 110 ++++++-- .../test/emqx_trace_SUITE.erl | 102 +++---- .../emqx_modules/src/emqx_mod_trace_api.erl | 14 +- .../test/emqx_mod_trace_api_SUITE.erl | 18 +- src/emqx_broker.erl | 9 +- src/emqx_channel.erl | 3 +- src/emqx_logger.erl | 49 ++-- src/emqx_trace_handler.erl | 206 ++++++++++++++ src/emqx_tracer.erl | 261 ------------------ test/emqx_trace_handler_SUITE.erl | 191 +++++++++++++ test/emqx_tracer_SUITE.erl | 171 ------------ 14 files changed, 840 insertions(+), 745 deletions(-) create mode 100644 src/emqx_trace_handler.erl delete mode 100644 src/emqx_tracer.erl create mode 100644 test/emqx_trace_handler_SUITE.erl delete mode 100644 test/emqx_tracer_SUITE.erl diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 97cb182e6..b6468fa7c 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -21,7 +21,7 @@ -include("emqx_mgmt.hrl"). --elvis([{elvis_style, invalid_dynamic_call, #{ ignore => [emqx_mgmt_cli]}}]). +-elvis([{elvis_style, invalid_dynamic_call, disable}]). -define(PRINT_CMD(Cmd, Desc), io:format("~-48s# ~s~n", [Cmd, Desc])). @@ -38,6 +38,7 @@ , vm/1 , mnesia/1 , trace/1 + , traces/1 , log/1 , mgmt/1 , data/1 @@ -421,39 +422,43 @@ log(_) -> trace(["list"]) -> lists:foreach(fun(Trace) -> - #{type := Type, level := Level, dst := Dst} = Trace, - Who = maps:get(Type, Trace), - emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Type, Who, Level, Dst]) - end, emqx_tracer:lookup_traces()); + #{type := Type, filter := Filter, level := Level, dst := Dst} = Trace, + emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Type, Filter, Level, Dst]) + end, emqx_trace_handler:running()); -trace(["stop", "client", ClientId]) -> - trace_off(clientid, ClientId); +trace(["stop", Operation, ClientId]) -> + case trace_type(Operation) of + {ok, Type} -> trace_off(Type, ClientId); + error -> trace([]) + end; -trace(["start", "client", ClientId, LogFile]) -> - trace_on(clientid, ClientId, all, LogFile); +trace(["start", Operation, ClientId, LogFile]) -> + trace(["start", Operation, ClientId, LogFile, "all"]); -trace(["start", "client", ClientId, LogFile, Level]) -> - trace_on(clientid, ClientId, list_to_atom(Level), LogFile); - -trace(["stop", "topic", Topic]) -> - trace_off(topic, Topic); - -trace(["start", "topic", Topic, LogFile]) -> - trace_on(topic, Topic, all, LogFile); - -trace(["start", "topic", Topic, LogFile, Level]) -> - trace_on(topic, Topic, list_to_atom(Level), LogFile); +trace(["start", Operation, ClientId, LogFile, Level]) -> + case trace_type(Operation) of + {ok, Type} -> trace_on(Type, ClientId, list_to_existing_atom(Level), LogFile); + error -> trace([]) + end; trace(_) -> - emqx_ctl:usage([{"trace list", "List all traces started"}, - {"trace start client []", "Traces for a client"}, - {"trace stop client ", "Stop tracing for a client"}, - {"trace start topic [] ", "Traces for a topic"}, - {"trace stop topic ", "Stop tracing for a topic"}]). + emqx_ctl:usage([{"trace list", "List all traces started on local node"}, + {"trace start client []", + "Traces for a client on local node"}, + {"trace stop client ", + "Stop tracing for a client on local node"}, + {"trace start topic [] ", + "Traces for a topic on local node"}, + {"trace stop topic ", + "Stop tracing for a topic on local node"}, + {"trace start ip_address [] ", + "Traces for a client ip on local node"}, + {"trace stop ip_addresss ", + "Stop tracing for a client ip on local node"} + ]). --dialyzer({nowarn_function, [trace_on/4, trace_off/2]}). trace_on(Who, Name, Level, LogFile) -> - case emqx_tracer:start_trace(Who, Name, Level, LogFile) of + case emqx_trace_handler:install(Who, Name, Level, LogFile) of ok -> emqx_ctl:print("trace ~s ~s successfully~n", [Who, Name]); {error, Error} -> @@ -461,13 +466,94 @@ trace_on(Who, Name, Level, LogFile) -> end. trace_off(Who, Name) -> - case emqx_tracer:stop_trace(Who, Name) of + case emqx_trace_handler:uninstall(Who, Name) of ok -> emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Name]); {error, Error} -> emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Who, Name, Error]) end. +%%-------------------------------------------------------------------- +%% @doc Trace Cluster Command +traces(["list"]) -> + {ok, List} = emqx_trace_api:list_trace(get, []), + case List of + [] -> + emqx_ctl:print("Cluster Trace is empty~n", []); + _ -> + lists:foreach(fun(Trace) -> + #{type := Type, name := Name, status := Status, + log_size := LogSize} = Trace, + emqx_ctl:print("Trace(~s: ~s=~s, ~s, LogSize:~p)~n", + [Name, Type, maps:get(Type, Trace), Status, LogSize]) + end, List) + end, + length(List); + +traces(["stop", Name]) -> + trace_cluster_off(Name); + +traces(["delete", Name]) -> + trace_cluster_del(Name); + +traces(["start", Name, Operation, Filter]) -> + traces(["start", Name, Operation, Filter, "900"]); + +traces(["start", Name, Operation, Filter, DurationS]) -> + case trace_type(Operation) of + {ok, Type} -> trace_cluster_on(Name, Type, Filter, DurationS); + error -> traces([]) + end; + +traces(_) -> + emqx_ctl:usage([{"traces list", "List all cluster traces started"}, + {"traces start client ", "Traces for a client in cluster"}, + {"traces start topic ", "Traces for a topic in cluster"}, + {"traces start ip_address ", "Traces for a IP in cluster"}, + {"traces stop ", "Stop trace in cluster"}, + {"traces delete ", "Delete trace in cluster"} + ]). + +trace_cluster_on(Name, Type, Filter, DurationS0) -> + case erlang:whereis(emqx_trace) of + undefined -> + emqx_ctl:print("[error] Tracer module not started~n" + "Please run `emqx_ctl modules start tracer` " + "or `emqx_ctl modules start emqx_mod_trace` first~n", []); + _ -> + DurationS = list_to_integer(DurationS0), + Now = erlang:system_time(second), + Trace = #{ name => list_to_binary(Name) + , type => atom_to_binary(Type) + , Type => list_to_binary(Filter) + , start_at => list_to_binary(calendar:system_time_to_rfc3339(Now)) + , end_at => list_to_binary(calendar:system_time_to_rfc3339(Now + DurationS)) + }, + case emqx_trace:create(Trace) of + ok -> + emqx_ctl:print("Cluster_trace ~p ~s ~s successfully~n", [Type, Filter, Name]); + {error, Error} -> + emqx_ctl:print("[error] Cluster_trace ~s ~s=~s ~p~n", + [Name, Type, Filter, Error]) + end + end. + +trace_cluster_del(Name) -> + case emqx_trace:delete(list_to_binary(Name)) of + ok -> emqx_ctl:print("Del cluster_trace ~s successfully~n", [Name]); + {error, Error} -> emqx_ctl:print("[error] Del cluster_trace ~s: ~p~n", [Name, Error]) + end. + +trace_cluster_off(Name) -> + case emqx_trace:update(list_to_binary(Name), false) of + ok -> emqx_ctl:print("Stop cluster_trace ~s successfully~n", [Name]); + {error, Error} -> emqx_ctl:print("[error] Stop cluster_trace ~s: ~p~n", [Name, Error]) + end. + +trace_type("client") -> {ok, clientid}; +trace_type("topic") -> {ok, topic}; +trace_type("ip_address") -> {ok, ip_address}; +trace_type(_) -> error. %%-------------------------------------------------------------------- %% @doc Listeners Command diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 77d46b744..434e96e21 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -45,6 +45,7 @@ groups() -> t_vm_cmd, t_plugins_cmd, t_trace_cmd, + t_traces_cmd, t_broker_cmd, t_router_cmd, t_subscriptions_cmd, @@ -64,6 +65,23 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_ct_helpers:stop_apps(apps()). +init_per_testcase(t_plugins_cmd, Config) -> + meck:new(emqx_plugins, [non_strict, passthrough]), + meck:expect(emqx_plugins, load, fun(_) -> ok end), + meck:expect(emqx_plugins, unload, fun(_) -> ok end), + meck:expect(emqx_plugins, reload, fun(_) -> ok end), + mock_print(), + Config; +init_per_testcase(_Case, Config) -> + mock_print(), + Config. + +end_per_testcase(t_plugins_cmd, _Config) -> + meck:unload(emqx_plugins), + unmock_print(); +end_per_testcase(_Case, _Config) -> + unmock_print(). + t_app(_Config) -> {ok, AppSecret} = emqx_mgmt_auth:add_app(<<"app_id">>, <<"app_name">>), ?assert(emqx_mgmt_auth:is_authorized(<<"app_id">>, AppSecret)), @@ -96,7 +114,6 @@ t_app(_Config) -> ok. t_log_cmd(_) -> - mock_print(), lists:foreach(fun(Level) -> emqx_mgmt_cli:log(["primary-level", Level]), ?assertEqual(Level ++ "\n", emqx_mgmt_cli:log(["primary-level"])) @@ -109,12 +126,9 @@ t_log_cmd(_) -> ?assertEqual(Level ++ "\n", emqx_mgmt_cli:log(["handlers", "set-level", atom_to_list(Id), Level])) end, ?LOG_LEVELS) - || #{id := Id} <- emqx_logger:get_log_handlers()], - meck:unload(). + || #{id := Id} <- emqx_logger:get_log_handlers()]. t_mgmt_cmd(_) -> - % ct:pal("start testing the mgmt command"), - mock_print(), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( ["lookup", "emqx_appid"]), "Not Found.")), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( @@ -127,28 +141,19 @@ t_mgmt_cmd(_) -> ["update", "emqx_appid", "ts"]), "update successfully")), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( ["delete", "emqx_appid"]), "ok")), - ok = emqx_mgmt_cli:mgmt(["list"]), - meck:unload(). + ok = emqx_mgmt_cli:mgmt(["list"]). t_status_cmd(_) -> - % ct:pal("start testing status command"), - mock_print(), %% init internal status seem to be always 'starting' when running ct tests - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([]), "Node\s.*@.*\sis\sstart(ed|ing)")), - meck:unload(). + ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([]), "Node\s.*@.*\sis\sstart(ed|ing)")). t_broker_cmd(_) -> - % ct:pal("start testing the broker command"), - mock_print(), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker([]), "sysdescr")), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker(["stats"]), "subscriptions.shared")), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker(["metrics"]), "bytes.sent")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker([undefined]), "broker")), - meck:unload(). + ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker([undefined]), "broker")). t_clients_cmd(_) -> - % ct:pal("start testing the client command"), - mock_print(), process_flag(trap_exit, true), {ok, T} = emqtt:start_link([{clientid, <<"client12">>}, {username, <<"testuser1">>}, @@ -164,7 +169,6 @@ t_clients_cmd(_) -> receive {'EXIT', T, _} -> ok - % ct:pal("Connection closed: ~p~n", [Reason]) after 500 -> erlang:error("Client is not kick") @@ -179,10 +183,11 @@ t_clients_cmd(_) -> {ok, Connack, <<>>, _} = raw_recv_pase(Bin), timer:sleep(300), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:clients(["show", "client13"]), "client13")), - meck:unload(). + % emqx_mgmt_cli:clients(["kick", "client13"]), % timer:sleep(500), % ?assertMatch({match, _}, re:run(emqx_mgmt_cli:clients(["show", "client13"]), "Not Found")). + ok. raw_recv_pase(Packet) -> emqx_frame:parse(Packet). @@ -191,8 +196,6 @@ raw_send_serialize(Packet) -> emqx_frame:serialize(Packet). t_vm_cmd(_) -> - % ct:pal("start testing the vm command"), - mock_print(), [[?assertMatch({match, _}, re:run(Result, Name)) || Result <- emqx_mgmt_cli:vm([Name])] || Name <- ["load", "memory", "process", "io", "ports"]], @@ -205,12 +208,9 @@ t_vm_cmd(_) -> [?assertMatch({match, _}, re:run(Result, "io")) || Result <- emqx_mgmt_cli:vm(["io"])], [?assertMatch({match, _}, re:run(Result, "ports")) - || Result <- emqx_mgmt_cli:vm(["ports"])], - unmock_print(). + || Result <- emqx_mgmt_cli:vm(["ports"])]. t_trace_cmd(_) -> - % ct:pal("start testing the trace command"), - mock_print(), logger:set_primary_config(level, debug), {ok, T} = emqtt:start_link([{clientid, <<"client">>}, {username, <<"testuser">>}, @@ -237,12 +237,34 @@ t_trace_cmd(_) -> Trace7 = emqx_mgmt_cli:trace(["start", "topic", "a/b/c", "log/clientid_trace.log", "error"]), ?assertMatch({match, _}, re:run(Trace7, "successfully")), - logger:set_primary_config(level, error), - unmock_print(). + logger:set_primary_config(level, error). + +t_traces_cmd(_) -> + emqx_trace:mnesia(boot), + Count1 = emqx_mgmt_cli:traces(["list"]), + ?assertEqual(0, Count1), + Error1 = emqx_mgmt_cli:traces(["start", "test-name", "client", "clientid-dev"]), + ?assertMatch({match, _}, re:run(Error1, "Tracer module not started")), + emqx_trace:start_link(), + Trace1 = emqx_mgmt_cli:traces(["start", "test-name", "client", "clientid-dev"]), + ?assertMatch({match, _}, re:run(Trace1, "successfully")), + Count2 = emqx_mgmt_cli:traces(["list"]), + ?assertEqual(1, Count2), + Error2 = emqx_mgmt_cli:traces(["start", "test-name", "client", "clientid-dev"]), + ?assertMatch({match, _}, re:run(Error2, "already_existed")), + Trace2 = emqx_mgmt_cli:traces(["stop", "test-name"]), + ?assertMatch({match, _}, re:run(Trace2, "successfully")), + Count3 = emqx_mgmt_cli:traces(["list"]), + ?assertEqual(1, Count3), + Trace3 = emqx_mgmt_cli:traces(["delete", "test-name"]), + ?assertMatch({match, _}, re:run(Trace3, "successfully")), + Count4 = emqx_mgmt_cli:traces(["list"]), + ?assertEqual(0, Count4), + Error3 = emqx_mgmt_cli:traces(["delete", "test-name"]), + ?assertMatch({match, _}, re:run(Error3, "not_found")), + ok. t_router_cmd(_) -> - % ct:pal("start testing the router command"), - mock_print(), {ok, T} = emqtt:start_link([{clientid, <<"client1">>}, {username, <<"testuser1">>}, {password, <<"pass1">>} @@ -257,12 +279,9 @@ t_router_cmd(_) -> emqtt:connect(T1), emqtt:subscribe(T1, <<"a/b/c/d">>), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:routes(["list"]), "a/b/c | a/b/c")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:routes(["show", "a/b/c"]), "a/b/c")), - unmock_print(). + ?assertMatch({match, _}, re:run(emqx_mgmt_cli:routes(["show", "a/b/c"]), "a/b/c")). t_subscriptions_cmd(_) -> - % ct:pal("Start testing the subscriptions command"), - mock_print(), {ok, T3} = emqtt:start_link([{clientid, <<"client">>}, {username, <<"testuser">>}, {password, <<"pass">>} @@ -273,22 +292,18 @@ t_subscriptions_cmd(_) -> [?assertMatch({match, _} , re:run(Result, "b/b/c")) || Result <- emqx_mgmt_cli:subscriptions(["show", <<"client">>])], ?assertEqual(emqx_mgmt_cli:subscriptions(["add", "client", "b/b/c", "0"]), "ok~n"), - ?assertEqual(emqx_mgmt_cli:subscriptions(["del", "client", "b/b/c"]), "ok~n"), - unmock_print(). + ?assertEqual(emqx_mgmt_cli:subscriptions(["del", "client", "b/b/c"]), "ok~n"). t_listeners_cmd_old(_) -> ok = emqx_listeners:ensure_all_started(), - mock_print(), ?assertEqual(emqx_mgmt_cli:listeners([]), ok), ?assertEqual( "Stop mqtt:wss:external listener on 0.0.0.0:8084 successfully.\n", emqx_mgmt_cli:listeners(["stop", "wss", "8084"]) - ), - unmock_print(). + ). t_listeners_cmd_new(_) -> ok = emqx_listeners:ensure_all_started(), - mock_print(), ?assertEqual(emqx_mgmt_cli:listeners([]), ok), ?assertEqual( "Stop mqtt:wss:external listener on 0.0.0.0:8084 successfully.\n", @@ -304,16 +319,11 @@ t_listeners_cmd_new(_) -> ), ?assertEqual( emqx_mgmt_cli:listeners(["restart", "bad:listener:identifier"]), - "Failed to restart bad:listener:identifier listener: {no_such_listener,\"bad:listener:identifier\"}\n" - ), - unmock_print(). + "Failed to restart bad:listener:identifier listener: " + "{no_such_listener,\"bad:listener:identifier\"}\n" + ). t_plugins_cmd(_) -> - mock_print(), - meck:new(emqx_plugins, [non_strict, passthrough]), - meck:expect(emqx_plugins, load, fun(_) -> ok end), - meck:expect(emqx_plugins, unload, fun(_) -> ok end), - meck:expect(emqx_plugins, reload, fun(_) -> ok end), ?assertEqual(emqx_mgmt_cli:plugins(["list"]), ok), ?assertEqual( emqx_mgmt_cli:plugins(["unload", "emqx_auth_mnesia"]), @@ -326,11 +336,9 @@ t_plugins_cmd(_) -> ?assertEqual( emqx_mgmt_cli:plugins(["unload", "emqx_management"]), "Plugin emqx_management can not be unloaded.~n" - ), - unmock_print(). + ). t_cli(_) -> - mock_print(), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([""]), "status")), [?assertMatch({match, _}, re:run(Value, "broker")) || Value <- emqx_mgmt_cli:broker([""])], @@ -352,9 +360,10 @@ t_cli(_) -> || Value <- emqx_mgmt_cli:mnesia([""])], [?assertMatch({match, _}, re:run(Value, "trace")) || Value <- emqx_mgmt_cli:trace([""])], + [?assertMatch({match, _}, re:run(Value, "traces")) + || Value <- emqx_mgmt_cli:traces([""])], [?assertMatch({match, _}, re:run(Value, "mgmt")) - || Value <- emqx_mgmt_cli:mgmt([""])], - unmock_print(). + || Value <- emqx_mgmt_cli:mgmt([""])]. mock_print() -> catch meck:unload(emqx_ctl), diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index eb2ed7276..eb41703fa 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -18,10 +18,9 @@ -behaviour(gen_server). -include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). --logger_header("[Trace]"). +-logger_header("[Tracer]"). %% Mnesia bootstrap -export([mnesia/1]). @@ -29,6 +28,11 @@ -boot_mnesia({mnesia, [boot]}). -copy_mnesia({mnesia, [copy]}). +-export([ publish/1 + , subscribe/3 + , unsubscribe/2 + ]). + -export([ start_link/0 , list/0 , list/1 @@ -41,6 +45,7 @@ -export([ format/1 , zip_dir/0 + , filename/2 , trace_dir/0 , trace_file/1 , delete_files_after_send/2 @@ -49,23 +54,22 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -define(TRACE, ?MODULE). --define(PACKETS, tuple_to_list(?TYPE_NAMES)). -define(MAX_SIZE, 30). -ifdef(TEST). -export([log_file/2]). -endif. +-export_type([ip_address/0]). +-type ip_address() :: string(). + -record(?TRACE, { name :: binary() | undefined | '_' - , type :: clientid | topic | undefined | '_' - , topic :: emqx_types:topic() | undefined | '_' - , clientid :: emqx_types:clientid() | undefined | '_' - , packets = [] :: list() | '_' + , type :: clientid | topic | ip_address | undefined | '_' + , filter :: emqx_types:topic() | emqx_types:clientid() | ip_address() | undefined | '_' , enable = true :: boolean() | '_' - , start_at :: integer() | undefined | binary() | '_' - , end_at :: integer() | undefined | binary() | '_' - , log_size = #{} :: map() | '_' + , start_at :: integer() | undefined | '_' + , end_at :: integer() | undefined | '_' }). mnesia(boot) -> @@ -77,6 +81,31 @@ mnesia(boot) -> mnesia(copy) -> ok = ekka_mnesia:copy_table(?TRACE, disc_copies). +publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore; +publish(#message{from = From, topic = Topic, payload = Payload}) when + is_binary(From); is_atom(From) -> + emqx_logger:info( + #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, + "PUBLISH to ~s: ~0p", + [Topic, Payload] + ). + +subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore; +subscribe(Topic, SubId, SubOpts) -> + emqx_logger:info( + #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, + "~ts SUBSCRIBE ~ts: Options: ~0p", + [SubId, Topic, SubOpts] + ). + +unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore; +unsubscribe(Topic, SubOpts) -> + emqx_logger:info( + #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, + "~ts UNSUBSCRIBE ~ts: Options: ~0p", + [maps:get(subid, SubOpts, ""), Topic, SubOpts] + ). + -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). @@ -89,18 +118,18 @@ list() -> list(Enable) -> ets:match_object(?TRACE, #?TRACE{enable = Enable, _ = '_'}). --spec create([{Key :: binary(), Value :: binary()}]) -> +-spec create([{Key :: binary(), Value :: binary()}] | #{atom() => binary()}) -> ok | {error, {duplicate_condition, iodata()} | {already_existed, iodata()} | iodata()}. create(Trace) -> case mnesia:table_info(?TRACE, size) < ?MAX_SIZE of true -> case to_trace(Trace) of - {ok, TraceRec} -> create_new_trace(TraceRec); + {ok, TraceRec} -> insert_new_trace(TraceRec); {error, Reason} -> {error, Reason} end; false -> - {error, """The number of traces created has reached the maximum, -please delete the useless ones first"""} + {error, "The number of traces created has reached the maximum" + " please delete the useless ones first"} end. -spec delete(Name :: binary()) -> ok | {error, not_found}. @@ -163,12 +192,8 @@ delete_files_after_send(TraceLog, Zips) -> -spec format(list(#?TRACE{})) -> list(map()). format(Traces) -> Fields = record_info(fields, ?TRACE), - lists:map(fun(Trace0 = #?TRACE{start_at = StartAt, end_at = EndAt}) -> - Trace = Trace0#?TRACE{ - start_at = list_to_binary(calendar:system_time_to_rfc3339(StartAt)), - end_at = list_to_binary(calendar:system_time_to_rfc3339(EndAt)) - }, - [_ | Values] = tuple_to_list(Trace), + lists:map(fun(Trace0 = #?TRACE{}) -> + [_ | Values] = tuple_to_list(Trace0), maps:from_list(lists:zip(Fields, Values)) end, Traces). @@ -198,7 +223,7 @@ handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitor case maps:take(Pid, Monitors) of error -> {noreply, State}; {Files, NewMonitors} -> - lists:foreach(fun(F) -> file:delete(F) end, Files), + lists:foreach(fun file:delete/1, Files), {noreply, State#{monitors => NewMonitors}} end; handle_info({timeout, TRef, update_trace}, @@ -227,14 +252,12 @@ terminate(_Reason, #{timer := TRef, primary_log_level := OriginLogLevel}) -> code_change(_, State, _Extra) -> {ok, State}. -create_new_trace(Trace) -> +insert_new_trace(Trace) -> Tran = fun() -> case mnesia:read(?TRACE, Trace#?TRACE.name) of [] -> - #?TRACE{start_at = StartAt, topic = Topic, - clientid = ClientId, packets = Packets} = Trace, - Match = #?TRACE{_ = '_', start_at = StartAt, topic = Topic, - clientid = ClientId, packets = Packets}, + #?TRACE{start_at = StartAt, type = Type, filter = Filter} = Trace, + Match = #?TRACE{_ = '_', start_at = StartAt, type = Type, filter = Filter}, case mnesia:match_object(?TRACE, Match, read) of [] -> mnesia:write(?TRACE, Trace, write); [#?TRACE{name = Name}] -> mnesia:abort({duplicate_condition, Name}) @@ -248,7 +271,7 @@ update_trace(Traces) -> Now = erlang:system_time(second), {_Waiting, Running, Finished} = classify_by_time(Traces, Now), disable_finished(Finished), - Started = already_running(), + Started = emqx_trace_handler:running(), {NeedRunning, AllStarted} = start_trace(Running, Started), NeedStop = AllStarted -- NeedRunning, ok = stop_trace(NeedStop, Started), @@ -257,14 +280,8 @@ update_trace(Traces) -> emqx_misc:start_timer(NextTime, update_trace). stop_all_trace_handler() -> - lists:foreach(fun(#{type := Type, name := Name} = Trace) -> - _ = emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name) - end - , already_running()). - -already_running() -> - emqx_tracer:lookup_traces(). - + lists:foreach(fun(#{id := Id}) -> emqx_trace_handler:uninstall(Id) end, + emqx_trace_handler:running()). get_enable_trace() -> {atomic, Traces} = mnesia:transaction(fun() -> @@ -286,31 +303,24 @@ find_closest_time(Traces, Now) -> disable_finished([]) -> ok; disable_finished(Traces) -> - NameWithLogSize = - lists:map(fun(#?TRACE{name = Name, start_at = StartAt}) -> - FileSize = filelib:file_size(log_file(Name, StartAt)), - {Name, FileSize} end, Traces), transaction(fun() -> - lists:map(fun({Name, LogSize}) -> + lists:map(fun(#?TRACE{name = Name}) -> case mnesia:read(?TRACE, Name, write) of [] -> ok; - [Trace = #?TRACE{log_size = Logs}] -> - mnesia:write(?TRACE, Trace#?TRACE{enable = false, - log_size = Logs#{node() => LogSize}}, write) - end end, NameWithLogSize) + [Trace] -> mnesia:write(?TRACE, Trace#?TRACE{enable = false}, write) + end end, Traces) end). start_trace(Traces, Started0) -> Started = lists:map(fun(#{name := Name}) -> Name end, Started0), lists:foldl(fun(#?TRACE{name = Name} = Trace, {Running, StartedAcc}) -> case lists:member(Name, StartedAcc) of - true -> {[Name | Running], StartedAcc}; + true -> + {[Name | Running], StartedAcc}; false -> case start_trace(Trace) of ok -> {[Name | Running], [Name | StartedAcc]}; - Error -> - ?LOG(error, "(~p)start trace failed by:~p", [Name, Error]), - {[Name | Running], StartedAcc} + {error, _Reason} -> {[Name | Running], StartedAcc} end end end, {[], Started}, Traces). @@ -318,27 +328,16 @@ start_trace(Traces, Started0) -> start_trace(Trace) -> #?TRACE{name = Name , type = Type - , clientid = ClientId - , topic = Topic - , packets = Packets + , filter = Filter , start_at = Start } = Trace, - Who0 = #{name => Name, labels => Packets}, - Who = - case Type of - topic -> Who0#{type => topic, topic => Topic}; - clientid -> Who0#{type => clientid, clientid => ClientId} - end, - case emqx_tracer:start_trace(Who, debug, log_file(Name, Start)) of - ok -> ok; - {error, {already_exist, _}} -> ok; - {error, Reason} -> {error, Reason} - end. + Who = #{name => Name, type => Type, filter => Filter}, + emqx_trace_handler:install(Who, debug, log_file(Name, Start)). stop_trace(Finished, Started) -> - lists:foreach(fun(#{name := Name, type := Type} = Trace) -> + lists:foreach(fun(#{name := Name, type := Type}) -> case lists:member(Name, Finished) of - true -> emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name); + true -> emqx_trace_handler:uninstall(Type, Name); false -> ok end end, Started). @@ -371,23 +370,31 @@ classify_by_time([Trace = #?TRACE{end_at = End} | Traces], classify_by_time([Trace | Traces], Now, Wait, Run, Finish) -> classify_by_time(Traces, Now, Wait, [Trace | Run], Finish). -to_trace(TraceList) -> - case to_trace(TraceList, #?TRACE{}) of +to_trace(TraceParam) -> + case to_trace(ensure_proplists(TraceParam), #?TRACE{}) of {error, Reason} -> {error, Reason}; {ok, #?TRACE{name = undefined}} -> {error, "name required"}; {ok, #?TRACE{type = undefined}} -> - {error, "type required"}; - {ok, #?TRACE{topic = undefined, clientid = undefined}} -> - {error, "topic/clientid cannot be both empty"}; - {ok, Trace} -> - case fill_default(Trace) of + {error, "type=[topic,clientid,ip_address] required"}; + {ok, #?TRACE{filter = undefined}} -> + {error, "topic/clientid/ip_address filter required"}; + {ok, TraceRec0} -> + case fill_default(TraceRec0) of #?TRACE{start_at = Start, end_at = End} when End =< Start -> {error, "failed by start_at >= end_at"}; - Trace1 -> {ok, Trace1} + TraceRec -> {ok, TraceRec} end end. +ensure_proplists(#{} = Trace) -> maps:to_list(Trace); +ensure_proplists(Trace) when is_list(Trace) -> + lists:foldl( + fun({K, V}, Acc) when is_binary(K) -> [{binary_to_existing_atom(K), V} | Acc]; + ({K, V}, Acc) when is_atom(K) -> [{K, V} | Acc]; + (_, Acc) -> Acc + end, [], Trace). + fill_default(Trace = #?TRACE{start_at = undefined}) -> fill_default(Trace#?TRACE{start_at = erlang:system_time(second)}); fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) -> @@ -395,27 +402,35 @@ fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) -> fill_default(Trace) -> Trace. to_trace([], Rec) -> {ok, Rec}; -to_trace([{<<"name">>, Name} | Trace], Rec) -> +to_trace([{name, Name} | Trace], Rec) -> case binary:match(Name, [<<"/">>], []) of - nomatch -> to_trace(Trace, Rec#?TRACE{name = Name}); + nomatch when byte_size(Name) < 200 -> to_trace(Trace, Rec#?TRACE{name = Name}); + nomatch -> {error, "name(latin1) length must < 200"}; _ -> {error, "name cannot contain /"} end; -to_trace([{<<"type">>, Type} | Trace], Rec) -> - case lists:member(Type, [<<"clientid">>, <<"topic">>]) of +to_trace([{type, Type} | Trace], Rec) -> + case lists:member(Type, [<<"clientid">>, <<"topic">>, <<"ip_address">>]) of true -> to_trace(Trace, Rec#?TRACE{type = binary_to_existing_atom(Type)}); - false -> {error, "incorrect type: only support clientid/topic"} + false -> {error, "incorrect type: only support clientid/topic/ip_address"} end; -to_trace([{<<"topic">>, Topic} | Trace], Rec) -> +to_trace([{topic, Topic} | Trace], Rec) -> case validate_topic(Topic) of - ok -> to_trace(Trace, Rec#?TRACE{topic = Topic}); + ok -> to_trace(Trace, Rec#?TRACE{filter = Topic}); {error, Reason} -> {error, Reason} end; -to_trace([{<<"start_at">>, StartAt} | Trace], Rec) -> +to_trace([{clientid, ClientId} | Trace], Rec) -> + to_trace(Trace, Rec#?TRACE{filter = ClientId}); +to_trace([{ip_address, IP} | Trace], Rec) -> + case inet:parse_address(binary_to_list(IP)) of + {ok, _} -> to_trace(Trace, Rec#?TRACE{filter = binary_to_list(IP)}); + {error, Reason} -> {error, lists:flatten(io_lib:format("ip address: ~p", [Reason]))} + end; +to_trace([{start_at, StartAt} | Trace], Rec) -> case to_system_second(StartAt) of {ok, Sec} -> to_trace(Trace, Rec#?TRACE{start_at = Sec}); {error, Reason} -> {error, Reason} end; -to_trace([{<<"end_at">>, EndAt} | Trace], Rec) -> +to_trace([{end_at, EndAt} | Trace], Rec) -> Now = erlang:system_time(second), case to_system_second(EndAt) of {ok, Sec} when Sec > Now -> @@ -425,21 +440,14 @@ to_trace([{<<"end_at">>, EndAt} | Trace], Rec) -> {error, Reason} -> {error, Reason} end; -to_trace([{<<"clientid">>, ClientId} | Trace], Rec) -> - to_trace(Trace, Rec#?TRACE{clientid = ClientId}); -to_trace([{<<"packets">>, PacketList} | Trace], Rec) -> - case to_packets(PacketList) of - {ok, Packets} -> to_trace(Trace, Rec#?TRACE{packets = Packets}); - {error, Reason} -> {error, io_lib:format("unsupport packets: ~p", [Reason])} - end; to_trace([Unknown | _Trace], _Rec) -> {error, io_lib:format("unknown field: ~p", [Unknown])}. validate_topic(TopicName) -> - try emqx_topic:validate(name, TopicName) of + try emqx_topic:validate(filter, TopicName) of true -> ok catch error:Error -> - {error, io_lib:format("~s invalid by ~p", [TopicName, Error])} + {error, io_lib:format("topic: ~s invalid by ~p", [TopicName, Error])} end. to_system_second(At) -> @@ -450,14 +458,6 @@ to_system_second(At) -> {error, ["The rfc3339 specification not satisfied: ", At]} end. -to_packets(Packets) when is_list(Packets) -> - AtomTypes = lists:map(fun(Type) -> binary_to_existing_atom(Type) end, Packets), - case lists:filter(fun(T) -> not lists:member(T, ?PACKETS) end, AtomTypes) of - [] -> {ok, AtomTypes}; - InvalidE -> {error, InvalidE} - end; -to_packets(Packets) -> {error, Packets}. - zip_dir() -> trace_dir() ++ "zip/". diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index e6c87d69f..7d982c00d 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -27,18 +27,37 @@ , download_zip_log/2 , stream_log_file/2 ]). --export([read_trace_file/3]). +-export([ read_trace_file/3 + , get_trace_size/0 + ]). -define(TO_BIN(_B_), iolist_to_binary(_B_)). --define(NOT_FOUND(N), {error, 'NOT_FOUND', ?TO_BIN([N, "NOT FOUND"])}). +-define(NOT_FOUND(N), {error, 'NOT_FOUND', ?TO_BIN([N, " NOT FOUND"])}). -list_trace(_, Params) -> - List = - case Params of - [{<<"enable">>, Enable}] -> emqx_trace:list(binary_to_existing_atom(Enable)); - _ -> emqx_trace:list() - end, - {ok, emqx_trace:format(List)}. +list_trace(_, _Params) -> + case emqx_trace:list() of + [] -> {ok, []}; + List0 -> + List = lists:sort(fun(#{start_at := A}, #{start_at := B}) -> A > B end, List0), + Nodes = ekka_mnesia:running_nodes(), + TraceSize = cluster_call(?MODULE, get_trace_size, [], 30000), + AllFileSize = lists:foldl(fun(F, Acc) -> maps:merge(Acc, F) end, #{}, TraceSize), + Now = erlang:system_time(second), + Traces = + lists:map(fun(Trace = #{name := Name, start_at := Start, + end_at := End, enable := Enable, type := Type, filter := Filter}) -> + FileName = emqx_trace:filename(Name, Start), + LogSize = collect_file_size(Nodes, FileName, AllFileSize), + Trace0 = maps:without([enable, filter], Trace), + Trace0#{ log_size => LogSize + , Type => Filter + , start_at => list_to_binary(calendar:system_time_to_rfc3339(Start)) + , end_at => list_to_binary(calendar:system_time_to_rfc3339(End)) + , status => status(Enable, Start, End, Now) + } + end, emqx_trace:format(List)), + {ok, Traces} + end. create_trace(_, Param) -> case emqx_trace:create(Param) of @@ -97,28 +116,48 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) -> end, [], TraceFiles). collect_trace_file(TraceLog) -> + cluster_call(emqx_trace, trace_file, [TraceLog], 60000). + +cluster_call(Mod, Fun, Args, Timeout) -> Nodes = ekka_mnesia:running_nodes(), - {Files, BadNodes} = rpc:multicall(Nodes, emqx_trace, trace_file, [TraceLog], 60000), - BadNodes =/= [] andalso ?LOG(error, "download log rpc failed on ~p", [BadNodes]), - Files. + {GoodRes, BadNodes} = rpc:multicall(Nodes, Mod, Fun, Args, Timeout), + BadNodes =/= [] andalso ?LOG(error, "rpc call failed on ~p ~p", [BadNodes, {Mod, Fun, Args}]), + GoodRes. stream_log_file(#{name := Name}, Params) -> Node0 = proplists:get_value(<<"node">>, Params, atom_to_binary(node())), Position0 = proplists:get_value(<<"position">>, Params, <<"0">>), Bytes0 = proplists:get_value(<<"bytes">>, Params, <<"1000">>), - Node = binary_to_existing_atom(Node0), - Position = binary_to_integer(Position0), - Bytes = binary_to_integer(Bytes0), - case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of - {ok, Bin} -> - Meta = #{<<"position">> => Position + byte_size(Bin), <<"bytes">> => Bytes}, - {ok, #{meta => Meta, items => Bin}}; - {eof, Size} -> - Meta = #{<<"position">> => Size, <<"bytes">> => Bytes}, - {ok, #{meta => Meta, items => <<"">>}}; - {error, Reason} -> - logger:log(error, "read_file_failed by ~p", [{Name, Reason, Position, Bytes}]), - {error, Reason} + case to_node(Node0) of + {ok, Node} -> + Position = binary_to_integer(Position0), + Bytes = binary_to_integer(Bytes0), + case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of + {ok, Bin} -> + Meta = #{<<"position">> => Position + byte_size(Bin), <<"bytes">> => Bytes}, + {ok, #{meta => Meta, items => Bin}}; + {eof, Size} -> + Meta = #{<<"position">> => Size, <<"bytes">> => Bytes}, + {ok, #{meta => Meta, items => <<"">>}}; + {error, Reason} -> + logger:log(error, "read_file_failed by ~p", [{Name, Reason, Position, Bytes}]), + {error, Reason}; + {badrpc, nodedown} -> + {error, "BadRpc node down"} + end; + {error, Reason} -> {error, Reason} + end. + +get_trace_size() -> + TraceDir = emqx_trace:trace_dir(), + Node = node(), + case file:list_dir(TraceDir) of + {ok, AllFiles} -> + lists:foldl(fun(File, Acc) -> + FullFileName = filename:join(TraceDir, File), + Acc#{{Node, File} => filelib:file_size(FullFileName)} + end, #{}, lists:delete("zip", AllFiles)); + _ -> #{} end. %% this is an rpc call for stream_log_file/2 @@ -134,7 +173,6 @@ read_trace_file(Name, Position, Limit) -> [] -> {error, not_found} end. --dialyzer({nowarn_function, read_file/3}). read_file(Path, Offset, Bytes) -> {ok, IoDevice} = file:open(Path, [read, raw, binary]), try @@ -146,9 +184,27 @@ read_file(Path, Offset, Bytes) -> {ok, Bin} -> {ok, Bin}; {error, Reason} -> {error, Reason}; eof -> - #file_info{size = Size} = file:read_file_info(IoDevice), + {ok, #file_info{size = Size}} = file:read_file_info(IoDevice), {eof, Size} end after file:close(IoDevice) end. + +to_node(Node) -> + try {ok, binary_to_existing_atom(Node)} + catch _:_ -> + {error, "node not found"} + end. + +collect_file_size(Nodes, FileName, AllFiles) -> + lists:foldl(fun(Node, Acc) -> + Size = maps:get({Node, FileName}, AllFiles, 0), + Acc#{Node => Size} + end, #{}, Nodes). + +%% status(false, _Start, End, Now) when End > Now -> <<"stopped">>; +status(false, _Start, _End, _Now) -> <<"stopped">>; +status(true, Start, _End, Now) when Now < Start -> <<"waiting">>; +status(true, _Start, End, Now) when Now >= End -> <<"stopped">>; +status(true, _Start, _End, _Now) -> <<"running">>. diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index 169fd50bc..ffa2bc1fb 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -23,21 +23,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). --record(emqx_trace, { - name, - type, - topic, - clientid, - packets = [], - enable = true, - start_at, - end_at, - log_size = #{} - }). - --define(PACKETS, ['CONNECT', 'CONNACK', 'PUBLISH', 'PUBACK', 'PUBREC', 'PUBREL' - , 'PUBCOMP', 'SUBSCRIBE', 'SUBACK', 'UNSUBSCRIBE', 'UNSUBACK' - , 'PINGREQ', 'PINGRESP', 'DISCONNECT', 'AUTH']). +-record(emqx_trace, {name, type, filter, enable = true, start_at, end_at}). %%-------------------------------------------------------------------- %% Setups @@ -61,16 +47,14 @@ t_base_create_delete(_Config) -> End = to_rfc3339(Now + 30 * 60), Name = <<"name1">>, ClientId = <<"test-device">>, - Packets = [atom_to_binary(E) || E <- ?PACKETS], - Trace = [ - {<<"name">>, Name}, - {<<"type">>, <<"clientid">>}, - {<<"clientid">>, ClientId}, - {<<"packets">>, Packets}, - {<<"start_at">>, Start}, - {<<"end_at">>, End} - ], - AnotherTrace = lists:keyreplace(<<"name">>, 1, Trace, {<<"name">>, <<"AnotherName">>}), + Trace = #{ + name => Name, + type => <<"clientid">>, + clientid => ClientId, + start_at => Start, + end_at => End + }, + AnotherTrace = Trace#{name => <<"anotherTrace">>}, ok = emqx_trace:create(Trace), ?assertEqual({error, {already_existed, Name}}, emqx_trace:create(Trace)), ?assertEqual({error, {duplicate_condition, Name}}, emqx_trace:create(AnotherTrace)), @@ -78,24 +62,19 @@ t_base_create_delete(_Config) -> Expect = #emqx_trace{ name = Name, type = clientid, - topic = undefined, - clientid = ClientId, - packets = ?PACKETS, + filter = ClientId, start_at = Now, end_at = Now + 30 * 60 }, ?assertEqual(Expect, TraceRec), ExpectFormat = [ #{ - clientid => <<"test-device">>, + filter => <<"test-device">>, enable => true, type => clientid, - packets => ?PACKETS, name => <<"name1">>, - start_at => Start, - end_at => End, - log_size => #{}, - topic => undefined + start_at => Now, + end_at => Now + 30 * 60 } ], ?assertEqual(ExpectFormat, emqx_trace:format([TraceRec])), @@ -108,13 +87,12 @@ t_create_size_max(_Config) -> emqx_trace:clear(), lists:map(fun(Seq) -> Name = list_to_binary("name" ++ integer_to_list(Seq)), - Trace = [{<<"name">>, Name}, {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, - {<<"topic">>, list_to_binary("/x/y/" ++ integer_to_list(Seq))}], + Trace = [{name, Name}, {type, <<"topic">>}, + {topic, list_to_binary("/x/y/" ++ integer_to_list(Seq))}], ok = emqx_trace:create(Trace) end, lists:seq(1, 30)), - Trace31 = [{<<"name">>, <<"name31">>}, {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/31">>}], + Trace31 = [{<<"name">>, <<"name31">>}, + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/31">>}], {error, _} = emqx_trace:create(Trace31), ok = emqx_trace:delete(<<"name30">>), ok = emqx_trace:create(Trace31), @@ -125,11 +103,11 @@ t_create_failed(_Config) -> ok = emqx_trace:clear(), UnknownField = [{<<"unknown">>, 12}], {error, Reason1} = emqx_trace:create(UnknownField), - ?assertEqual(<<"unknown field: {<<\"unknown\">>,12}">>, iolist_to_binary(Reason1)), + ?assertEqual(<<"unknown field: {unknown,12}">>, iolist_to_binary(Reason1)), InvalidTopic = [{<<"topic">>, "#/#//"}], {error, Reason2} = emqx_trace:create(InvalidTopic), - ?assertEqual(<<"#/#// invalid by function_clause">>, iolist_to_binary(Reason2)), + ?assertEqual(<<"topic: #/#// invalid by function_clause">>, iolist_to_binary(Reason2)), InvalidStart = [{<<"start_at">>, <<"2021-12-3:12">>}], {error, Reason3} = emqx_trace:create(InvalidStart), @@ -141,41 +119,34 @@ t_create_failed(_Config) -> ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, iolist_to_binary(Reason4)), - InvalidPackets = [{<<"packets">>, [<<"publish">>]}], - {error, Reason5} = emqx_trace:create(InvalidPackets), - ?assertEqual(<<"unsupport packets: [publish]">>, iolist_to_binary(Reason5)), - - InvalidPackets2 = [{<<"packets">>, <<"publish">>}], - {error, Reason6} = emqx_trace:create(InvalidPackets2), - ?assertEqual(<<"unsupport packets: <<\"publish\">>">>, iolist_to_binary(Reason6)), - {error, Reason7} = emqx_trace:create([{<<"name">>, <<"test">>}, {<<"type">>, <<"clientid">>}]), - ?assertEqual(<<"topic/clientid cannot be both empty">>, iolist_to_binary(Reason7)), + ?assertEqual(<<"topic/clientid/ip_address filter required">>, iolist_to_binary(Reason7)), InvalidPackets4 = [{<<"name">>, <<"/test">>}, {<<"clientid">>, <<"t">>}, {<<"type">>, <<"clientid">>}], {error, Reason9} = emqx_trace:create(InvalidPackets4), ?assertEqual(<<"name cannot contain /">>, iolist_to_binary(Reason9)), - ?assertEqual({error, "type required"}, emqx_trace:create([{<<"name">>, <<"test-name">>}, - {<<"packets">>, []}, {<<"clientid">>, <<"good">>}])), + ?assertEqual({error, "type=[topic,clientid,ip_address] required"}, + emqx_trace:create([{<<"name">>, <<"test-name">>}, {<<"clientid">>, <<"good">>}])), - ?assertEqual({error, "incorrect type: only support clientid/topic"}, + ?assertEqual({error, "incorrect type: only support clientid/topic/ip_address"}, emqx_trace:create([{<<"name">>, <<"test-name">>}, - {<<"packets">>, []}, {<<"clientid">>, <<"good">>}, {<<"type">>, <<"typeerror">> }])), + {<<"clientid">>, <<"good">>}, {<<"type">>, <<"typeerror">> }])), + + ?assertEqual({error, "ip address: einval"}, + emqx_trace:create([{<<"ip_address">>, <<"test-name">>}])), ok. t_create_default(_Config) -> ok = emqx_trace:clear(), {error, "name required"} = emqx_trace:create([]), ok = emqx_trace:create([{<<"name">>, <<"test-name">>}, - {<<"type">>, <<"clientid">>}, {<<"packets">>, []}, {<<"clientid">>, <<"good">>}]), - [#emqx_trace{packets = Packets}] = emqx_trace:list(), - ?assertEqual([], Packets), + {<<"type">>, <<"clientid">>}, {<<"clientid">>, <<"good">>}]), + [#emqx_trace{name = <<"test-name">>}] = emqx_trace:list(), ok = emqx_trace:clear(), Trace = [ {<<"name">>, <<"test-name">>}, - {<<"packets">>, [<<"PUBLISH">>]}, {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/z">>}, {<<"start_at">>, <<"2021-10-28T10:54:47+08:00">>}, @@ -186,15 +157,13 @@ t_create_default(_Config) -> Trace2 = [ {<<"name">>, <<"test-name">>}, {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/z">>}, {<<"start_at">>, to_rfc3339(Now + 10)}, {<<"end_at">>, to_rfc3339(Now + 3)} ], {error, "failed by start_at >= end_at"} = emqx_trace:create(Trace2), ok = emqx_trace:create([{<<"name">>, <<"test-name">>}, - {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/z">>}]), + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/z">>}]), [#emqx_trace{start_at = Start, end_at = End}] = emqx_trace:list(), ?assertEqual(10 * 60, End - Start), ?assertEqual(true, Start - erlang:system_time(second) < 5), @@ -205,8 +174,8 @@ t_update_enable(_Config) -> Name = <<"test-name">>, Now = erlang:system_time(second), End = list_to_binary(calendar:system_time_to_rfc3339(Now + 2)), - ok = emqx_trace:create([{<<"name">>, Name}, {<<"packets">>, [<<"PUBLISH">>]}, - {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/z">>}, {<<"end_at">>, End}]), + ok = emqx_trace:create([{<<"name">>, Name}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/z">>}, {<<"end_at">>, End}]), [#emqx_trace{enable = Enable}] = emqx_trace:list(), ?assertEqual(Enable, true), ok = emqx_trace:update(Name, false), @@ -293,13 +262,10 @@ t_get_log_filename(_Config) -> Start = calendar:system_time_to_rfc3339(Now), End = calendar:system_time_to_rfc3339(Now + 2), Name = <<"name1">>, - ClientId = <<"test-device">>, - Packets = [atom_to_binary(E) || E <- ?PACKETS], Trace = [ {<<"name">>, Name}, - {<<"type">>, <<"clientid">>}, - {<<"clientid">>, ClientId}, - {<<"packets">>, Packets}, + {<<"type">>, <<"ip_address">>}, + {<<"ip_address">>, <<"127.0.0.1">>}, {<<"start_at">>, list_to_binary(Start)}, {<<"end_at">>, list_to_binary(End)} ], diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl index 1daab1520..0b2963af6 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl @@ -19,7 +19,7 @@ %% API -export([ list_trace/2 , create_trace/2 - , update_trace/2 + , disable_trace/2 , delete_trace/2 , clear_traces/2 , download_zip_log/2 @@ -52,11 +52,11 @@ func => clear_traces, descr => "clear all traces"}). --rest_api(#{name => update_trace, +-rest_api(#{name => disable_trace, method => 'PUT', - path => "/trace/:bin:name/:atom:operation", - func => update_trace, - descr => "diable/enable trace"}). + path => "/trace/:bin:name/stop", + func => disable_trace, + descr => "stop trace"}). -rest_api(#{name => download_zip_log, method => 'GET', @@ -82,8 +82,8 @@ delete_trace(Path, Params) -> clear_traces(Path, Params) -> return(emqx_trace_api:clear_traces(Path, Params)). -update_trace(Path, Params) -> - return(emqx_trace_api:update_trace(Path, Params)). +disable_trace(#{name := Name}, Params) -> + return(emqx_trace_api:update_trace(#{name => Name, operation => disable}, Params)). download_zip_log(Path, Params) -> case emqx_trace_api:download_zip_log(Path, Params) of diff --git a/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl index fc786dbd0..36ceb8c49 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl @@ -52,14 +52,13 @@ t_http_test(_Config) -> %% create ErrorTrace = #{}, {ok, Error} = request_api(post, api_path("trace"), Header, ErrorTrace), - ?assertEqual(#{<<"message">> => <<"unknown field: {}">>, + ?assertEqual(#{<<"message">> => <<"name required">>, <<"code">> => <<"INCORRECT_PARAMS">>}, json(Error)), Name = <<"test-name">>, Trace = [ {<<"name">>, Name}, {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/z">>} ], @@ -71,14 +70,23 @@ t_http_test(_Config) -> ?assertEqual(Name, maps:get(<<"name">>, Data)), %% update - {ok, Update} = request_api(put, api_path("trace/test-name/disable"), Header, #{}), + {ok, Update} = request_api(put, api_path("trace/test-name/stop"), Header, #{}), ?assertEqual(#{<<"code">> => 0, <<"data">> => #{<<"enable">> => false, <<"name">> => <<"test-name">>}}, json(Update)), {ok, List1} = request_api(get, api_path("trace"), Header), #{<<"code">> := 0, <<"data">> := [Data1]} = json(List1), - ?assertEqual(false, maps:get(<<"enable">>, Data1)), + Node = atom_to_binary(node()), + ?assertMatch(#{ + <<"status">> := <<"stopped">>, + <<"name">> := <<"test-name">>, + <<"log_size">> := #{Node := _}, + <<"start_at">> := _, + <<"end_at">> := _, + <<"type">> := <<"topic">>, + <<"topic">> := <<"/x/y/z">> + }, Data1), %% delete {ok, Delete} = request_api(delete, api_path("trace/test-name"), Header), @@ -86,7 +94,7 @@ t_http_test(_Config) -> {ok, DeleteNotFound} = request_api(delete, api_path("trace/test-name"), Header), ?assertEqual(#{<<"code">> => <<"NOT_FOUND">>, - <<"message">> => <<"test-nameNOT FOUND">>}, json(DeleteNotFound)), + <<"message">> => <<"test-name NOT FOUND">>}, json(DeleteNotFound)), {ok, List2} = request_api(get, api_path("trace"), Header), ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(List2)), diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index 14617e2e1..bcf343432 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -126,8 +126,9 @@ subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) -> -spec(subscribe(emqx_topic:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok). subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) -> SubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts0), - _ = emqx_tracer:trace_subscribe(Topic, SubId, SubOpts), - case ets:member(?SUBOPTION, {SubPid = self(), Topic}) of + _ = emqx_trace:subscribe(Topic, SubId, SubOpts), + SubPid = self(), + case ets:member(?SUBOPTION, {SubPid, Topic}) of false -> %% New ok = emqx_broker_helper:register_sub(SubPid, SubId), do_subscribe(Topic, SubPid, with_subid(SubId, SubOpts)); @@ -172,7 +173,7 @@ unsubscribe(Topic) when is_binary(Topic) -> SubPid = self(), case ets:lookup(?SUBOPTION, {SubPid, Topic}) of [{_, SubOpts}] -> - emqx_tracer:trace_unsubscribe(Topic, SubOpts), + emqx_trace:unsubscribe(Topic, SubOpts), _ = emqx_broker_helper:reclaim_seq(Topic), do_unsubscribe(Topic, SubPid, SubOpts); [] -> ok @@ -195,7 +196,7 @@ do_unsubscribe(Group, Topic, SubPid, _SubOpts) -> -spec(publish(emqx_types:message()) -> emqx_types:publish_result()). publish(Msg) when is_record(Msg, message) -> - _ = emqx_tracer:trace_publish(Msg), + _ = emqx_trace:publish(Msg), emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'), case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of #message{headers = #{allow_publish := false}} -> diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 4c49a1cb3..bad053845 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -275,7 +275,7 @@ take_ws_cookie(ClientInfo, ConnInfo) -> handle_in(?CONNECT_PACKET(), Channel = #channel{conn_state = connected}) -> handle_out(disconnect, ?RC_PROTOCOL_ERROR, Channel); -handle_in(?CONNECT_PACKET(ConnPkt), Channel) -> +handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) -> case pipeline([fun enrich_conninfo/2, fun run_conn_hooks/2, fun check_connect/2, @@ -285,6 +285,7 @@ handle_in(?CONNECT_PACKET(ConnPkt), Channel) -> fun auth_connect/2 ], ConnPkt, Channel#channel{conn_state = connecting}) of {ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} -> + ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]), NChannel1 = NChannel#channel{ will_msg = emqx_packet:will_msg(NConnPkt), alias_maximum = init_alias_maximum(NConnPkt, ClientInfo) diff --git a/src/emqx_logger.erl b/src/emqx_logger.erl index a733b0d3a..9cf7050ea 100644 --- a/src/emqx_logger.erl +++ b/src/emqx_logger.erl @@ -18,6 +18,8 @@ -compile({no_auto_import, [error/1]}). +-elvis([{elvis_style, god_modules, disable}]). + %% Logs -export([ debug/1 , debug/2 @@ -64,10 +66,11 @@ id := logger:handler_id(), level := logger:level(), dst := logger_dst(), + filters := [{logger:filter_id(), logger:filter()}], status := started | stopped }). --define(stopped_handlers, {?MODULE, stopped_handlers}). +-define(STOPPED_HANDLERS, {?MODULE, stopped_handlers}). %%-------------------------------------------------------------------- %% APIs @@ -171,19 +174,19 @@ get_log_handlers() -> -spec(get_log_handlers(started | stopped) -> [logger_handler_info()]). get_log_handlers(started) -> - [log_hanlder_info(Conf, started) || Conf <- logger:get_handler_config()]; + [log_handler_info(Conf, started) || Conf <- logger:get_handler_config()]; get_log_handlers(stopped) -> - [log_hanlder_info(Conf, stopped) || Conf <- list_stopped_handler_config()]. + [log_handler_info(Conf, stopped) || Conf <- list_stopped_handler_config()]. -spec(get_log_handler(logger:handler_id()) -> logger_handler_info()). get_log_handler(HandlerId) -> case logger:get_handler_config(HandlerId) of {ok, Conf} -> - log_hanlder_info(Conf, started); + log_handler_info(Conf, started); {error, _} -> case read_stopped_handler_config(HandlerId) of error -> {error, {not_found, HandlerId}}; - {ok, Conf} -> log_hanlder_info(Conf, stopped) + {ok, Conf} -> log_handler_info(Conf, stopped) end end. @@ -245,21 +248,21 @@ parse_transform(AST, _Opts) -> %% Internal Functions %%-------------------------------------------------------------------- -log_hanlder_info(#{id := Id, level := Level, module := logger_std_h, - config := #{type := Type}}, Status) when +log_handler_info(#{id := Id, level := Level, module := logger_std_h, + filters := Filters, config := #{type := Type}}, Status) when Type =:= standard_io; Type =:= standard_error -> - #{id => Id, level => Level, dst => console, status => Status}; -log_hanlder_info(#{id := Id, level := Level, module := logger_std_h, - config := Config = #{type := file}}, Status) -> - #{id => Id, level => Level, status => Status, + #{id => Id, level => Level, dst => console, status => Status, filters => Filters}; +log_handler_info(#{id := Id, level := Level, module := logger_std_h, + filters := Filters, config := Config = #{type := file}}, Status) -> + #{id => Id, level => Level, status => Status, filters => Filters, dst => maps:get(file, Config, atom_to_list(Id))}; -log_hanlder_info(#{id := Id, level := Level, module := logger_disk_log_h, - config := #{file := Filename}}, Status) -> - #{id => Id, level => Level, dst => Filename, status => Status}; -log_hanlder_info(#{id := Id, level := Level, module := _OtherModule}, Status) -> - #{id => Id, level => Level, dst => unknown, status => Status}. +log_handler_info(#{id := Id, level := Level, module := logger_disk_log_h, + filters := Filters, config := #{file := Filename}}, Status) -> + #{id => Id, level => Level, dst => Filename, status => Status, filters => Filters}; +log_handler_info(#{id := Id, level := Level, filters := Filters}, Status) -> + #{id => Id, level => Level, dst => unknown, status => Status, filters => Filters}. %% set level for all log handlers in one command set_all_log_handlers_level(Level) -> @@ -281,29 +284,29 @@ rollback([{ID, Level} | List]) -> rollback([]) -> ok. save_stopped_handler_config(HandlerId, Config) -> - case persistent_term:get(?stopped_handlers, undefined) of + case persistent_term:get(?STOPPED_HANDLERS, undefined) of undefined -> - persistent_term:put(?stopped_handlers, #{HandlerId => Config}); + persistent_term:put(?STOPPED_HANDLERS, #{HandlerId => Config}); ConfList -> - persistent_term:put(?stopped_handlers, ConfList#{HandlerId => Config}) + persistent_term:put(?STOPPED_HANDLERS, ConfList#{HandlerId => Config}) end. read_stopped_handler_config(HandlerId) -> - case persistent_term:get(?stopped_handlers, undefined) of + case persistent_term:get(?STOPPED_HANDLERS, undefined) of undefined -> error; ConfList -> maps:find(HandlerId, ConfList) end. remove_stopped_handler_config(HandlerId) -> - case persistent_term:get(?stopped_handlers, undefined) of + case persistent_term:get(?STOPPED_HANDLERS, undefined) of undefined -> ok; ConfList -> case maps:find(HandlerId, ConfList) of error -> ok; {ok, _} -> - persistent_term:put(?stopped_handlers, maps:remove(HandlerId, ConfList)) + persistent_term:put(?STOPPED_HANDLERS, maps:remove(HandlerId, ConfList)) end end. list_stopped_handler_config() -> - case persistent_term:get(?stopped_handlers, undefined) of + case persistent_term:get(?STOPPED_HANDLERS, undefined) of undefined -> []; ConfList -> maps:values(ConfList) end. diff --git a/src/emqx_trace_handler.erl b/src/emqx_trace_handler.erl new file mode 100644 index 000000000..e4008405a --- /dev/null +++ b/src/emqx_trace_handler.erl @@ -0,0 +1,206 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_trace_handler). + +-include("emqx.hrl"). +-include("logger.hrl"). + +-logger_header("[Tracer]"). + +%% APIs +-export([ running/0 + , install/3 + , install/4 + , uninstall/1 + , uninstall/2 + ]). + +%% For logger handler filters callbacks +-export([ filter_clientid/2 + , filter_topic/2 + , filter_ip_address/2 + ]). + +-type tracer() :: #{ + name := binary(), + type := clientid | topic | ip_address, + filter := emqx_types:clientid() | emqx_types:topic() | emqx_trace:ip_address() + }. + +-define(FORMAT, + {logger_formatter, #{ + template => [ + time, " [", level, "] ", + {clientid, + [{peername, [clientid, "@", peername, " "], [clientid, " "]}], + [{peername, [peername, " "], []}] + }, + msg, "\n" + ], + single_line => false, + max_size => unlimited, + depth => unlimited + }} +). + +-define(CONFIG(_LogFile_), #{ + type => halt, + file => _LogFile_, + max_no_bytes => 512 * 1024 * 1024, + overload_kill_enable => true, + overload_kill_mem_size => 50 * 1024 * 1024, + overload_kill_qlen => 20000, + %% disable restart + overload_kill_restart_after => infinity + }). + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +-spec install(Name :: binary() | list(), + Type :: clientid | topic | ip_address, + Filter ::emqx_types:clientid() | emqx_types:topic() | string(), + Level :: logger:level() | all, + LogFilePath :: string()) -> ok | {error, term()}. +install(Name, Type, Filter, Level, LogFile) -> + Who = #{type => Type, filter => ensure_bin(Filter), name => ensure_bin(Name)}, + install(Who, Level, LogFile). + +-spec install(Type :: clientid | topic | ip_address, + Filter ::emqx_types:clientid() | emqx_types:topic() | string(), + Level :: logger:level() | all, + LogFilePath :: string()) -> ok | {error, term()}. +install(Type, Filter, Level, LogFile) -> + install(Filter, Type, Filter, Level, LogFile). + +-spec install(tracer(), logger:level() | all, string()) -> ok | {error, term()}. +install(Who, all, LogFile) -> + install(Who, debug, LogFile); +install(Who, Level, LogFile) -> + PrimaryLevel = emqx_logger:get_primary_log_level(), + try logger:compare_levels(Level, PrimaryLevel) of + lt -> + {error, + io_lib:format( + "Cannot trace at a log level (~s) " + "lower than the primary log level (~s)", + [Level, PrimaryLevel] + )}; + _GtOrEq -> + install_handler(Who, Level, LogFile) + catch + error:badarg -> + {error, {invalid_log_level, Level}} + end. + +-spec uninstall(Type :: clientid | topic | ip_address, + Name :: binary() | list()) -> ok | {error, term()}. +uninstall(Type, Name) -> + HandlerId = handler_id(#{type => Type, name => ensure_bin(Name)}), + uninstall(HandlerId). + +-spec uninstall(HandlerId :: atom()) -> ok | {error, term()}. +uninstall(HandlerId) -> + Res = logger:remove_handler(HandlerId), + show_prompts(Res, HandlerId, "Stop trace"), + Res. + +%% @doc Return all running trace handlers information. +-spec running() -> + [ + #{ + name => binary(), + type => topic | clientid | ip_address, + id => atom(), + filter => emqx_types:topic() | emqx_types:clienetid() | emqx_trace:ip_address(), + level => logger:level(), + dst => file:filename() | console | unknown + } + ]. +running() -> + lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)). + +-spec filter_clientid(logger:log_event(), string()) -> logger:log_event() | ignore. +filter_clientid(#{meta := #{clientid := ClientId}} = Log, ClientId) -> Log; +filter_clientid(_Log, _ExpectId) -> ignore. + +-spec filter_topic(logger:log_event(), string()) -> logger:log_event() | ignore. +filter_topic(#{meta := #{topic := Topic}} = Log, TopicFilter) -> + case emqx_topic:match(Topic, TopicFilter) of + true -> Log; + false -> ignore + end; +filter_topic(_Log, _ExpectId) -> ignore. + +-spec filter_ip_address(logger:log_event(), string()) -> logger:log_event() | ignore. +filter_ip_address(#{meta := #{peername := Peername}} = Log, IP) -> + case lists:prefix(IP, Peername) of + true -> Log; + false -> ignore + end; +filter_ip_address(_Log, _ExpectId) -> ignore. + +install_handler(Who, Level, LogFile) -> + HandlerId = handler_id(Who), + Config = #{ + level => Level, + formatter => ?FORMAT, + filter_default => stop, + filters => filters(Who), + config => ?CONFIG(LogFile) + }, + Res = logger:add_handler(HandlerId, logger_disk_log_h, Config), + show_prompts(Res, Who, "Start trace"), + Res. + +filters(#{type := clientid, filter := Filter}) -> + [{clientid, {fun ?MODULE:filter_clientid/2, ensure_list(Filter)}}]; +filters(#{type := topic, filter := Filter}) -> + [{topic, {fun ?MODULE:filter_topic/2, ensure_bin(Filter)}}]; +filters(#{type := ip_address, filter := Filter}) -> + [{ip_address, {fun ?MODULE:filter_ip_address/2, ensure_list(Filter)}}]. + +filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) -> + Init = #{id => Id, level => Level, dst => Dst}, + case Filters of + [{topic, {_FilterFun, Filter}}] -> + <<"trace_topic_", Name/binary>> = atom_to_binary(Id), + [Init#{type => topic, filter => Filter, name => Name} | Acc]; + [{clientid, {_FilterFun, Filter}}] -> + <<"trace_clientid_", Name/binary>> = atom_to_binary(Id), + [Init#{type => clientid, filter => Filter, name => Name} | Acc]; + [{ip_address, {_FilterFun, Filter}}] -> + <<"trace_ip_address_", Name/binary>> = atom_to_binary(Id), + [Init#{type => ip_address, filter => Filter, name => Name} | Acc]; + _ -> + Acc + end. + +handler_id(#{type := Type, name := Name}) -> + binary_to_atom(<<"trace_", (atom_to_binary(Type))/binary, "_", Name/binary>>). + +ensure_bin(List) when is_list(List) -> iolist_to_binary(List); +ensure_bin(Bin) when is_binary(Bin) -> Bin. + +ensure_list(Bin) when is_binary(Bin) -> binary_to_list(Bin); +ensure_list(List) when is_list(List) -> List. + +show_prompts(ok, Who, Msg) -> + ?LOG(info, Msg ++ " ~p " ++ "successfully~n", [Who]); +show_prompts({error, Reason}, Who, Msg) -> + ?LOG(error, Msg ++ " ~p " ++ "failed by ~p~n", [Who, Reason]). diff --git a/src/emqx_tracer.erl b/src/emqx_tracer.erl deleted file mode 100644 index e563c8eab..000000000 --- a/src/emqx_tracer.erl +++ /dev/null @@ -1,261 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_tracer). - --include("emqx.hrl"). --include("logger.hrl"). - --logger_header("[Tracer]"). - -%% APIs --export([ trace_publish/1 - , trace_subscribe/3 - , trace_unsubscribe/2 - , start_trace/3 - , start_trace/4 - , lookup_traces/0 - , stop_trace/3 - , stop_trace/2 - ]). - --ifdef(TEST). --export([is_match/3]). --endif. - --type(label() :: 'CONNECT' | 'CONNACK' | 'PUBLISH' | 'PUBACK' | 'PUBREC' | - 'PUBREL' | 'PUBCOMP' | 'SUBSCRIBE' | 'SUBACK' | 'UNSUBSCRIBE' | - 'UNSUBACK' | 'PINGREQ' | 'PINGRESP' | 'DISCONNECT' | 'AUTH'). - --type(tracer() :: #{name := binary(), - type := clientid | topic, - clientid => emqx_types:clientid(), - topic => emqx_types:topic(), - labels := [label()]}). - --define(TRACER, ?MODULE). --define(FORMAT, {logger_formatter, - #{template => - [time, " [", level, "] ", - {clientid, - [{peername, - [clientid, "@", peername, " "], - [clientid, " "]}], - [{peername, - [peername, " "], - []}]}, - msg, "\n"], - single_line => false - }}). --define(TOPIC_COMBINATOR, <<"_trace_topic_">>). --define(CLIENTID_COMBINATOR, <<"_trace_clientid_">>). --define(TOPIC_TRACE_ID(T, N), - binary_to_atom(<<(N)/binary, ?TOPIC_COMBINATOR/binary, (T)/binary>>)). --define(CLIENT_TRACE_ID(C, N), - binary_to_atom(<<(N)/binary, ?CLIENTID_COMBINATOR/binary, (C)/binary>>)). --define(TOPIC_TRACE(T, N, M), {topic, T, N, M}). --define(CLIENT_TRACE(C, N, M), {clientid, C, N, M}). --define(TOPIC_TRACE(T, N), {topic, T, N}). --define(CLIENT_TRACE(C, N), {clientid, C, N}). - --define(IS_LOG_LEVEL(L), - L =:= emergency orelse - L =:= alert orelse - L =:= critical orelse - L =:= error orelse - L =:= warning orelse - L =:= notice orelse - L =:= info orelse - L =:= debug). - -%%------------------------------------------------------------------------------ -%% APIs -%%------------------------------------------------------------------------------ -trace_publish(#message{topic = <<"$SYS/", _/binary>>}) -> - %% Do not trace '$SYS' publish - ignore; -trace_publish(#message{from = From, topic = Topic, payload = Payload}) - when is_binary(From); is_atom(From) -> - emqx_logger:info(#{topic => Topic, - mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} }, - "PUBLISH to ~s: ~0p", [Topic, Payload]). - -trace_subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore; -trace_subscribe(Topic, SubId, SubOpts) -> - emqx_logger:info(#{topic => Topic, - mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, - "~ts SUBSCRIBE ~ts: Options: ~0p", [SubId, Topic, SubOpts]). - -trace_unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore; -trace_unsubscribe(Topic, SubOpts) -> - emqx_logger:info(#{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, - "~ts UNSUBSCRIBE ~ts: Options: ~0p", - [maps:get(subid, SubOpts, ""), Topic, SubOpts]). - --spec(start_trace(clientid | topic, emqx_types:clientid() | emqx_types:topic(), - logger:level() | all, string()) -> ok | {error, term()}). -start_trace(clientid, ClientId0, Level, LogFile) -> - ClientId = ensure_bin(ClientId0), - Who = #{type => clientid, clientid => ClientId, name => ClientId, labels => []}, - start_trace(Who, Level, LogFile); -start_trace(topic, Topic0, Level, LogFile) -> - Topic = ensure_bin(Topic0), - Who = #{type => topic, topic => Topic, name => Topic, labels => []}, - start_trace(Who, Level, LogFile). - -%% @doc Start to trace clientid or topic. --spec(start_trace(tracer(), logger:level() | all, string()) -> ok | {error, term()}). -start_trace(Who, all, LogFile) -> - start_trace(Who, debug, LogFile); -start_trace(Who, Level, LogFile) -> - case ?IS_LOG_LEVEL(Level) of - true -> - PrimaryLevel = emqx_logger:get_primary_log_level(), - try logger:compare_levels(Level, PrimaryLevel) of - lt -> - {error, - io_lib:format("Cannot trace at a log level (~s) " - "lower than the primary log level (~s)", - [Level, PrimaryLevel])}; - _GtOrEq -> - install_trace_handler(Who, Level, LogFile) - catch - _:Error -> - {error, Error} - end; - false -> {error, {invalid_log_level, Level}} - end. - --spec(stop_trace(clientid | topic, emqx_types:clientid() | emqx_types:topic()) -> - ok | {error, term()}). -stop_trace(Type, ClientIdOrTopic) -> - stop_trace(Type, ClientIdOrTopic, ClientIdOrTopic). - -%% @doc Stop tracing clientid or topic. --spec(stop_trace(clientid | topic, emqx_types:clientid() | emqx_types:topic(), binary()) -> - ok | {error, term()}). -stop_trace(clientid, ClientId, Name) -> - Who = #{type => clientid, clientid => ensure_bin(ClientId), name => ensure_bin(Name)}, - uninstall_trance_handler(Who); -stop_trace(topic, Topic, Name) -> - Who = #{type => topic, topic => ensure_bin(Topic), name => ensure_bin(Name)}, - uninstall_trance_handler(Who). - -%% @doc Lookup all traces --spec(lookup_traces() -> [#{ type => topic | clientid, - name => binary(), - topic => emqx_types:topic(), - level => logger:level(), - dst => file:filename() | console | unknown - }]). -lookup_traces() -> - lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)). - -install_trace_handler(Who, Level, LogFile) -> - case logger:add_handler(handler_id(Who), logger_disk_log_h, - #{level => Level, - formatter => ?FORMAT, - config => #{type => halt, file => LogFile}, - filter_default => stop, - filters => [{meta_key_filter, - {fun filter_by_meta_key/2, Who}}]}) - of - ok -> - ?LOG(info, "Start trace for ~p", [Who]), - ok; - {error, Reason} -> - ?LOG(error, "Start trace for ~p failed, error: ~p", [Who, Reason]), - {error, Reason} - end. - -uninstall_trance_handler(Who) -> - case logger:remove_handler(handler_id(Who)) of - ok -> - ?LOG(info, "Stop trace for ~p", [Who]), - ok; - {error, Reason} -> - ?LOG(error, "Stop trace for ~p failed, error: ~p", [Who, Reason]), - {error, Reason} - end. - -filter_traces(#{id := Id, level := Level, dst := Dst}, Acc) -> - IdStr = atom_to_binary(Id), - case binary:split(IdStr, [?TOPIC_COMBINATOR]) of - [Name, Topic] -> - [#{ type => topic, - name => Name, - topic => Topic, - level => Level, - dst => Dst} | Acc]; - _ -> - case binary:split(IdStr, [?CLIENTID_COMBINATOR]) of - [Name, ClientId] -> - [#{ type => clientid, - name => Name, - clientid => ClientId, - level => Level, - dst => Dst} | Acc]; - _ -> Acc - end - end. - -%% Plan to support topic_and_client type, so we have type field. -handler_id(#{type := topic, topic := Topic, name := Name}) -> - ?TOPIC_TRACE_ID(format(Topic), format(Name)); -handler_id(#{type := clientid, clientid := ClientId, name := Name}) -> - ?CLIENT_TRACE_ID(format(ClientId), format(Name)). - -filter_by_meta_key(#{meta := Meta, level := Level} = Log, Context) -> - case is_match(Context, Meta, Level) of - true -> Log; - false -> ignore - end. - -%% When the log level is higher than debug and clientid/topic is match, -%% it will be logged without judging the content inside the labels. -%% When the log level is debug, in addition to the matched clientid/topic, -%% you also need to determine whether the label is in the labels -is_match(#{type := clientid, clientid := ExpectId, labels := Labels}, - #{clientid := RealId} = Meta, - Level) -> - is_match(ExpectId =:= iolist_to_binary(RealId), Level, Meta, Labels); -is_match(#{type := topic, topic := TopicFilter, labels := Labels}, - #{topic := Topic} = Meta, Level) -> - is_match(emqx_topic:match(Topic, TopicFilter), Level, Meta, Labels); -is_match(_, _, _) -> - false. - -is_match(true, debug, Meta, Labels) -> is_match_labels(Meta, Labels); -is_match(Boolean, _, _Meta, _Labels) -> Boolean. - -is_match_labels(#{trace_label := 'ALL'}, _Context) -> true; -is_match_labels(_, []) -> true; -is_match_labels(#{trace_label := Packet}, Context) -> - lists:member(Packet, Context); -is_match_labels(_, _) -> false. - -format(List)when is_list(List) -> - format(list_to_binary(List)); -format(Atom)when is_atom(Atom) -> - format(atom_to_list(Atom)); -format(Bin0)when is_binary(Bin0) -> - case byte_size(Bin0) of - Size when Size =< 200 -> Bin0; - _ -> emqx_misc:bin2hexstr_a_f_upper(Bin0) - end. - -ensure_bin(List) when is_list(List) -> iolist_to_binary(List); -ensure_bin(Bin) when is_binary(Bin) -> Bin. diff --git a/test/emqx_trace_handler_SUITE.erl b/test/emqx_trace_handler_SUITE.erl new file mode 100644 index 000000000..5e909c45b --- /dev/null +++ b/test/emqx_trace_handler_SUITE.erl @@ -0,0 +1,191 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_trace_handler_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("common_test/include/ct.hrl"). +-define(CLIENT, [{host, "localhost"}, + {clientid, <<"client">>}, + {username, <<"testuser">>}, + {password, <<"pass">>} + ]). + +all() -> [t_trace_clientid, t_trace_topic, t_trace_ip_address]. + +init_per_suite(Config) -> + emqx_ct_helpers:boot_modules(all), + emqx_ct_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([]). + +init_per_testcase(t_trace_clientid, Config) -> + Config; +init_per_testcase(_Case, Config) -> + ok = emqx_logger:set_log_level(debug), + _ = [logger:remove_handler(Id) ||#{id := Id} <- emqx_trace_handler:running()], + Config. + +end_per_testcase(_Case, _Config) -> + ok = emqx_logger:set_log_level(warning), + ok. + +t_trace_clientid(_Config) -> + %% Start tracing + emqx_logger:set_log_level(error), + {error, _} = emqx_trace_handler:install(clientid, <<"client">>, debug, "tmp/client.log"), + emqx_logger:set_log_level(debug), + %% add list clientid + ok = emqx_trace_handler:install(clientid, "client", debug, "tmp/client.log"), + ok = emqx_trace_handler:install(clientid, <<"client2">>, all, "tmp/client2.log"), + ok = emqx_trace_handler:install(clientid, <<"client3">>, all, "tmp/client3.log"), + {error, {invalid_log_level, bad_level}} = + emqx_trace_handler:install(clientid, <<"client4">>, bad_level, "tmp/client4.log"), + {error, {handler_not_added, {file_error, ".", eisdir}}} = + emqx_trace_handler:install(clientid, <<"client5">>, debug, "."), + ct:sleep(100), + + %% Verify the tracing file exits + ?assert(filelib:is_regular("tmp/client.log")), + ?assert(filelib:is_regular("tmp/client2.log")), + ?assert(filelib:is_regular("tmp/client3.log")), + + %% Get current traces + ?assertEqual([#{type => clientid, filter => "client", name => <<"client">>, + id => trace_clientid_client, level => debug, dst => "tmp/client.log"}, + #{type => clientid, filter => "client2", name => <<"client2">>, + id => trace_clientid_client2, level => debug, dst => "tmp/client2.log"}, + #{type => clientid, filter => "client3", name => <<"client3">>, + id => trace_clientid_client3, level => debug, dst => "tmp/client3.log"} + ], emqx_trace_handler:running()), + + %% Client with clientid = "client" publishes a "hi" message to "a/b/c". + {ok, T} = emqtt:start_link(?CLIENT), + emqtt:connect(T), + emqtt:publish(T, <<"a/b/c">>, <<"hi">>), + emqtt:ping(T), + ct:sleep(200), + + %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log". + {ok, Bin} = file:read_file("tmp/client.log"), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"CONNECT">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"CONNACK">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"PUBLISH">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"PINGREQ">>])), + ?assert(filelib:file_size("tmp/client2.log") == 0), + + %% Stop tracing + ok = emqx_trace_handler:uninstall(clientid, <<"client">>), + ok = emqx_trace_handler:uninstall(clientid, <<"client2">>), + ok = emqx_trace_handler:uninstall(clientid, <<"client3">>), + + emqtt:disconnect(T), + ?assertEqual([], emqx_trace_handler:running()). + +t_trace_topic(_Config) -> + {ok, T} = emqtt:start_link(?CLIENT), + emqtt:connect(T), + + %% Start tracing + emqx_logger:set_log_level(debug), + ok = emqx_trace_handler:install(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"), + ok = emqx_trace_handler:install(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"), + ct:sleep(100), + + %% Verify the tracing file exits + ?assert(filelib:is_regular("tmp/topic_trace_x.log")), + ?assert(filelib:is_regular("tmp/topic_trace_y.log")), + + %% Get current traces + ?assertEqual([#{type => topic, filter => <<"x/#">>, id => 'trace_topic_x/#', + level => debug, dst => "tmp/topic_trace_x.log", name => <<"x/#">>}, + #{type => topic, filter => <<"y/#">>, id => 'trace_topic_y/#', + name => <<"y/#">>, level => debug, dst => "tmp/topic_trace_y.log"} + ], + emqx_trace_handler:running()), + + %% Client with clientid = "client" publishes a "hi" message to "x/y/z". + emqtt:publish(T, <<"x/y/z">>, <<"hi1">>), + emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), + emqtt:subscribe(T, <<"x/y/z">>), + emqtt:unsubscribe(T, <<"x/y/z">>), + ct:sleep(200), + + {ok, Bin} = file:read_file("tmp/topic_trace_x.log"), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi2">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"PUBLISH">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"SUBSCRIBE">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"UNSUBSCRIBE">>])), + ?assert(filelib:file_size("tmp/topic_trace_y.log") =:= 0), + + %% Stop tracing + ok = emqx_trace_handler:uninstall(topic, <<"x/#">>), + ok = emqx_trace_handler:uninstall(topic, <<"y/#">>), + {error, _Reason} = emqx_trace_handler:uninstall(topic, <<"z/#">>), + ?assertEqual([], emqx_trace_handler:running()), + emqtt:disconnect(T). + +t_trace_ip_address(_Config) -> + {ok, T} = emqtt:start_link(?CLIENT), + emqtt:connect(T), + + %% Start tracing + ok = emqx_trace_handler:install(ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"), + ok = emqx_trace_handler:install(ip_address, "192.168.1.1", all, "tmp/ip_trace_y.log"), + ct:sleep(100), + + %% Verify the tracing file exits + ?assert(filelib:is_regular("tmp/ip_trace_x.log")), + ?assert(filelib:is_regular("tmp/ip_trace_y.log")), + + %% Get current traces + ?assertEqual([#{type => ip_address, filter => "127.0.0.1", + id => 'trace_ip_address_127.0.0.1', name => <<"127.0.0.1">>, + level => debug, dst => "tmp/ip_trace_x.log"}, + #{type => ip_address, filter => "192.168.1.1", + id => 'trace_ip_address_192.168.1.1', name => <<"192.168.1.1">>, + level => debug, dst => "tmp/ip_trace_y.log"} + ], + emqx_trace_handler:running()), + + %% Client with clientid = "client" publishes a "hi" message to "x/y/z". + emqtt:publish(T, <<"x/y/z">>, <<"hi1">>), + emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), + emqtt:subscribe(T, <<"x/y/z">>), + emqtt:unsubscribe(T, <<"x/y/z">>), + ct:sleep(200), + + {ok, Bin} = file:read_file("tmp/ip_trace_x.log"), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi2">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"PUBLISH">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"SUBSCRIBE">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"UNSUBSCRIBE">>])), + ?assert(filelib:file_size("tmp/ip_trace_y.log") =:= 0), + + %% Stop tracing + ok = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.1">>), + ok = emqx_trace_handler:uninstall(ip_address, <<"192.168.1.1">>), + {error, _Reason} = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.2">>), + emqtt:disconnect(T), + ?assertEqual([], emqx_trace_handler:running()). diff --git a/test/emqx_tracer_SUITE.erl b/test/emqx_tracer_SUITE.erl deleted file mode 100644 index 5f7105774..000000000 --- a/test/emqx_tracer_SUITE.erl +++ /dev/null @@ -1,171 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_tracer_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - --include_lib("common_test/include/ct.hrl"). --define(CLIENT, [{host, "localhost"}, - {clientid, <<"client">>}, - {username, <<"testuser">>}, - {password, <<"pass">>} - ]). - -all() -> [t_trace_clientid, t_trace_topic, t_is_match]. - -init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), - Config. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). - -t_trace_clientid(_Config) -> - {ok, T} = emqtt:start_link(?CLIENT), - emqtt:connect(T), - - %% Start tracing - emqx_logger:set_log_level(error), - {error, _} = emqx_tracer:start_trace(clientid, <<"client">>, debug, "tmp/client.log"), - emqx_logger:set_log_level(debug), - %% add list clientid - ok = emqx_tracer:start_trace(clientid, "client", debug, "tmp/client.log"), - ok = emqx_tracer:start_trace(clientid, <<"client2">>, all, "tmp/client2.log"), - ok = emqx_tracer:start_trace(clientid, <<"client3">>, all, "tmp/client3.log"), - {error, {invalid_log_level, bad_level}} = - emqx_tracer:start_trace(clientid, <<"client4">>, bad_level, "tmp/client4.log"), - {error, {handler_not_added, {file_error, ".", eisdir}}} = - emqx_tracer:start_trace(clientid, <<"client5">>, debug, "."), - ct:sleep(100), - - %% Verify the tracing file exits - ?assert(filelib:is_regular("tmp/client.log")), - ?assert(filelib:is_regular("tmp/client2.log")), - ?assert(filelib:is_regular("tmp/client3.log")), - - %% Get current traces - ?assertEqual([#{type => clientid, clientid => <<"client">>, - name => <<"client">>, level => debug, dst => "tmp/client.log"}, - #{type => clientid, clientid => <<"client2">>, - name => <<"client2">>, level => debug, dst => "tmp/client2.log"}, - #{type => clientid, clientid => <<"client3">>, - name => <<"client3">>, level => debug, dst => "tmp/client3.log"} - ], emqx_tracer:lookup_traces()), - - %% set the overall log level to debug - emqx_logger:set_log_level(debug), - - %% Client with clientid = "client" publishes a "hi" message to "a/b/c". - emqtt:publish(T, <<"a/b/c">>, <<"hi">>), - ct:sleep(200), - - %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log". - ?assert(filelib:file_size("tmp/client.log") > 0), - ?assert(filelib:file_size("tmp/client2.log") == 0), - - %% Stop tracing - ok = emqx_tracer:stop_trace(clientid, <<"client">>), - ok = emqx_tracer:stop_trace(clientid, <<"client2">>), - ok = emqx_tracer:stop_trace(clientid, <<"client3">>), - emqtt:disconnect(T), - - emqx_logger:set_log_level(warning). - -t_trace_topic(_Config) -> - {ok, T} = emqtt:start_link(?CLIENT), - emqtt:connect(T), - - %% Start tracing - emqx_logger:set_log_level(debug), - ok = emqx_tracer:start_trace(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"), - ok = emqx_tracer:start_trace(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"), - ct:sleep(100), - - %% Verify the tracing file exits - ?assert(filelib:is_regular("tmp/topic_trace_x.log")), - ?assert(filelib:is_regular("tmp/topic_trace_y.log")), - - %% Get current traces - ?assertEqual([#{type => topic, topic => <<"x/#">>, name => <<"x/#">>, - level => debug, dst => "tmp/topic_trace_x.log"}, - #{type => topic, topic => <<"y/#">>, name => <<"y/#">>, - level => debug, dst => "tmp/topic_trace_y.log"}], - emqx_tracer:lookup_traces()), - - %% set the overall log level to debug - emqx_logger:set_log_level(debug), - - %% Client with clientid = "client" publishes a "hi" message to "x/y/z". - emqtt:publish(T, <<"x/y/z">>, <<"hi1">>), - emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), - ct:sleep(200), - - ?assert(filelib:file_size("tmp/topic_trace_x.log") > 0), - ?assert(filelib:file_size("tmp/topic_trace_y.log") =:= 0), - - %% Stop tracing - ok = emqx_tracer:stop_trace(topic, <<"x/#">>), - ok = emqx_tracer:stop_trace(topic, <<"y/#">>), - {error, _Reason} = emqx_tracer:stop_trace(topic, <<"z/#">>), - emqtt:disconnect(T), - - emqx_logger:set_log_level(warning). - -t_is_match(_Config) -> - ClientId = <<"test">>, - ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, - #{clientid => ClientId}, warning)), - ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, - #{clientid => ClientId}, debug)), - ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, - labels => ['PUBLISH']}, #{clientid => ClientId}, debug)), - ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, - #{clientid => ClientId, trace_label => 'PUBLISH'}, debug)), - ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => ['PUBLISH']}, - #{clientid => ClientId, trace_label => 'PUBLISH'}, debug)), - ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => ['SUBACK']}, - #{clientid => ClientId, trace_label => 'PUBLISH'}, debug)), - ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => ['SUBACK']}, - #{clientid => ClientId, trace_label => 'ALL'}, debug)), - ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, - #{clientid => <<"Bad">>}, warning)), - ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, - #{clientid => <<"Bad">>, trace_label => 'PUBLISH'}, debug)), - - Topic = <<"/test/#">>, - ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, - #{topic => <<"/test/1">>}, warning)), - ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, - #{topic => <<"/test/1/2">>}, debug)), - ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['SUBSCRIBE']}, - #{topic => <<"/test/1/2">>}, debug)), - ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, - #{topic => <<"/test/3">>, trace_label => 'PUBLISH'}, debug)), - ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['PUBLISH']}, - #{topic => <<"/test/398/">>, trace_label => 'PUBLISH'}, debug)), - ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['SUBACK']}, - #{topic => <<"/test/1/xy/y">>, trace_label => 'PUBLISH'}, debug)), - - ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['PUBLISH']}, - #{topic => <<"/t1est/398/">>, trace_label => 'PUBLISH'}, debug)), - ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, - #{topic => <<"/t1est/1/xy/y">>, trace_label => 'PUBLISH'}, debug)), - ok. From a91f975dc2a77d8fc9a37095208cfcba3c284954 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 22 Nov 2021 18:08:48 +0800 Subject: [PATCH 087/104] fix: make sure keepalive only 0~65535 (#6232) --- apps/emqx_management/src/emqx_mgmt.erl | 6 ++++-- apps/emqx_management/src/emqx_mgmt_api_clients.erl | 1 + apps/emqx_management/test/emqx_mgmt_api_SUITE.erl | 9 +++++++++ src/emqx_keepalive.erl | 14 +++++++++++++- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 93f24cd37..61c32613b 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -274,8 +274,10 @@ set_ratelimit_policy(ClientId, Policy) -> set_quota_policy(ClientId, Policy) -> call_client(ClientId, {quota, Policy}). -set_keepalive(ClientId, Interval) -> - call_client(ClientId, {keepalive, Interval}). +set_keepalive(ClientId, Interval)when Interval >= 0 andalso Interval =< 65535 -> + call_client(ClientId, {keepalive, Interval}); +set_keepalive(_ClientId, _Interval) -> + {error, ?ERROR2, <<"mqtt3.1.1 specification: keepalive must between 0~65535">>}. %% @private call_client(ClientId, Req) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 5aacc7895..a25cc9ff5 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -261,6 +261,7 @@ set_keepalive(#{clientid := ClientId}, Params) -> case emqx_mgmt:set_keepalive(emqx_mgmt_util:urldecode(ClientId), Interval) of ok -> minirest:return(); {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); + {error, Code, Reason} -> minirest:return({error, Code, Reason}); {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) end end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index f3bc42f3a..a3eea2ab3 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -657,6 +657,15 @@ t_keepalive(_Config) -> {ok, _} = emqtt:connect(C1), {ok, Ok} = request_api(put, Path, "interval=5", AuthHeader, [#{}]), ?assertEqual("{\"code\":0}", Ok), + [Pid] = emqx_cm:lookup_channels(list_to_binary(ClientId)), + #{conninfo := #{keepalive := Keepalive}} = emqx_connection:info(Pid), + ?assertEqual(5, Keepalive), + {ok, Error1} = request_api(put, Path, "interval=-1", AuthHeader, [#{}]), + {ok, Error2} = request_api(put, Path, "interval=65536", AuthHeader, [#{}]), + ErrMsg = #{<<"code">> => 102, + <<"message">> => <<"mqtt3.1.1 specification: keepalive must between 0~65535">>}, + ?assertEqual(ErrMsg, jiffy:decode(Error1, [return_maps])), + ?assertEqual(Error1, Error2), emqtt:disconnect(C1), application:stop(emqx_dashboard), ok. diff --git a/src/emqx_keepalive.erl b/src/emqx_keepalive.erl index 216999acc..bef0714f9 100644 --- a/src/emqx_keepalive.erl +++ b/src/emqx_keepalive.erl @@ -73,7 +73,19 @@ check(NewVal, KeepAlive = #keepalive{statval = OldVal, true -> {error, timeout} end. +%% from mqtt-v3.1.1 specific +%% A Keep Alive value of zero (0) has the effect of turning off the keep alive mechanism. +%% This means that, in this case, the Server is not required +%% to disconnect the Client on the grounds of inactivity. +%% Note that a Server is permitted to disconnect a Client that it determines +%% to be inactive or non-responsive at any time, +%% regardless of the Keep Alive value provided by that Client. +%% Non normative comment +%%The actual value of the Keep Alive is application specific; +%% typically this is a few minutes. +%% The maximum value is (65535s) 18 hours 12 minutes and 15 seconds. + %% @doc Update keepalive's interval -spec(set(interval, non_neg_integer(), keepalive()) -> keepalive()). -set(interval, Interval, KeepAlive) -> +set(interval, Interval, KeepAlive) when Interval >= 0 andalso Interval =< 65535000 -> KeepAlive#keepalive{interval = Interval}. From 30fb9dd7ae8c2684170155295eaf6b00335a353d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 22 Nov 2021 20:40:56 +0800 Subject: [PATCH 088/104] fix: name must be printable unicode and len < 256 --- .../src/emqx_trace/emqx_trace.erl | 28 +++--- src/emqx_trace_handler.erl | 90 +++++++++++-------- test/emqx_trace_handler_SUITE.erl | 32 +++---- 3 files changed, 86 insertions(+), 64 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index eb41703fa..2190e5bab 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -128,7 +128,7 @@ create(Trace) -> {error, Reason} -> {error, Reason} end; false -> - {error, "The number of traces created has reached the maximum" + {error, "The number of traces created has reache the maximum" " please delete the useless ones first"} end. @@ -379,11 +379,16 @@ to_trace(TraceParam) -> {error, "type=[topic,clientid,ip_address] required"}; {ok, #?TRACE{filter = undefined}} -> {error, "topic/clientid/ip_address filter required"}; - {ok, TraceRec0} -> - case fill_default(TraceRec0) of - #?TRACE{start_at = Start, end_at = End} when End =< Start -> - {error, "failed by start_at >= end_at"}; - TraceRec -> {ok, TraceRec} + {ok, TraceRec0 = #?TRACE{name = Name, type = Type}} -> + case emqx_trace_handler:handler_id(Name, Type) of + {ok, _} -> + case fill_default(TraceRec0) of + #?TRACE{start_at = Start, end_at = End} when End =< Start -> + {error, "failed by start_at >= end_at"}; + TraceRec -> {ok, TraceRec} + end; + {error, Reason} -> + {error, Reason} end end. @@ -403,10 +408,13 @@ fill_default(Trace) -> Trace. to_trace([], Rec) -> {ok, Rec}; to_trace([{name, Name} | Trace], Rec) -> - case binary:match(Name, [<<"/">>], []) of - nomatch when byte_size(Name) < 200 -> to_trace(Trace, Rec#?TRACE{name = Name}); - nomatch -> {error, "name(latin1) length must < 200"}; - _ -> {error, "name cannot contain /"} + case io_lib:printable_unicode_list(binary_to_list(Name)) of + true -> + case binary:match(Name, [<<"/">>], []) of + nomatch -> to_trace(Trace, Rec#?TRACE{name = Name}); + _ -> {error, "name cannot contain /"} + end; + false -> {error, "name must printable unicode"} end; to_trace([{type, Type} | Trace], Rec) -> case lists:member(Type, [<<"clientid">>, <<"topic">>, <<"ip_address">>]) of diff --git a/src/emqx_trace_handler.erl b/src/emqx_trace_handler.erl index e4008405a..7006fc815 100644 --- a/src/emqx_trace_handler.erl +++ b/src/emqx_trace_handler.erl @@ -35,6 +35,8 @@ , filter_ip_address/2 ]). +-export([handler_id/2]). + -type tracer() :: #{ name := binary(), type := clientid | topic | ip_address, @@ -111,8 +113,10 @@ install(Who, Level, LogFile) -> -spec uninstall(Type :: clientid | topic | ip_address, Name :: binary() | list()) -> ok | {error, term()}. uninstall(Type, Name) -> - HandlerId = handler_id(#{type => Type, name => ensure_bin(Name)}), - uninstall(HandlerId). + case handler_id(ensure_bin(Name), Type) of + {ok, HandlerId} -> uninstall(HandlerId); + {error, Reason} -> {error, Reason} + end. -spec uninstall(HandlerId :: atom()) -> ok | {error, term()}. uninstall(HandlerId) -> @@ -135,64 +139,74 @@ uninstall(HandlerId) -> running() -> lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)). --spec filter_clientid(logger:log_event(), string()) -> logger:log_event() | ignore. -filter_clientid(#{meta := #{clientid := ClientId}} = Log, ClientId) -> Log; +-spec filter_clientid(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. +filter_clientid(#{meta := #{clientid := ClientId}} = Log, {ClientId, _Name}) -> Log; filter_clientid(_Log, _ExpectId) -> ignore. --spec filter_topic(logger:log_event(), string()) -> logger:log_event() | ignore. -filter_topic(#{meta := #{topic := Topic}} = Log, TopicFilter) -> +-spec filter_topic(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. +filter_topic(#{meta := #{topic := Topic}} = Log, {TopicFilter, _Name}) -> case emqx_topic:match(Topic, TopicFilter) of true -> Log; false -> ignore end; filter_topic(_Log, _ExpectId) -> ignore. --spec filter_ip_address(logger:log_event(), string()) -> logger:log_event() | ignore. -filter_ip_address(#{meta := #{peername := Peername}} = Log, IP) -> +-spec filter_ip_address(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. +filter_ip_address(#{meta := #{peername := Peername}} = Log, {IP, _Name}) -> case lists:prefix(IP, Peername) of true -> Log; false -> ignore end; filter_ip_address(_Log, _ExpectId) -> ignore. -install_handler(Who, Level, LogFile) -> - HandlerId = handler_id(Who), - Config = #{ - level => Level, - formatter => ?FORMAT, - filter_default => stop, - filters => filters(Who), - config => ?CONFIG(LogFile) - }, - Res = logger:add_handler(HandlerId, logger_disk_log_h, Config), - show_prompts(Res, Who, "Start trace"), - Res. +install_handler(Who = #{name := Name, type := Type}, Level, LogFile) -> + case handler_id(Name, Type) of + {ok, HandlerId} -> + Config = #{ + level => Level, + formatter => ?FORMAT, + filter_default => stop, + filters => filters(Who), + config => ?CONFIG(LogFile) + }, + Res = logger:add_handler(HandlerId, logger_disk_log_h, Config), + show_prompts(Res, Who, "Start trace"), + Res; + {error, _Reason} = Error -> + show_prompts(Error, Who, "Start trace"), + Error + end. -filters(#{type := clientid, filter := Filter}) -> - [{clientid, {fun ?MODULE:filter_clientid/2, ensure_list(Filter)}}]; -filters(#{type := topic, filter := Filter}) -> - [{topic, {fun ?MODULE:filter_topic/2, ensure_bin(Filter)}}]; -filters(#{type := ip_address, filter := Filter}) -> - [{ip_address, {fun ?MODULE:filter_ip_address/2, ensure_list(Filter)}}]. +filters(#{type := clientid, filter := Filter, name := Name}) -> + [{clientid, {fun ?MODULE:filter_clientid/2, {ensure_list(Filter), Name}}}]; +filters(#{type := topic, filter := Filter, name := Name}) -> + [{topic, {fun ?MODULE:filter_topic/2, {ensure_bin(Filter), Name}}}]; +filters(#{type := ip_address, filter := Filter, name := Name}) -> + [{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}]. filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) -> Init = #{id => Id, level => Level, dst => Dst}, case Filters of - [{topic, {_FilterFun, Filter}}] -> - <<"trace_topic_", Name/binary>> = atom_to_binary(Id), - [Init#{type => topic, filter => Filter, name => Name} | Acc]; - [{clientid, {_FilterFun, Filter}}] -> - <<"trace_clientid_", Name/binary>> = atom_to_binary(Id), - [Init#{type => clientid, filter => Filter, name => Name} | Acc]; - [{ip_address, {_FilterFun, Filter}}] -> - <<"trace_ip_address_", Name/binary>> = atom_to_binary(Id), - [Init#{type => ip_address, filter => Filter, name => Name} | Acc]; + [{Type, {_FilterFun, {Filter, Name}}}] when + Type =:= topic orelse + Type =:= clientid orelse + Type =:= ip_address -> + [Init#{type => Type, filter => Filter, name => Name} | Acc]; _ -> Acc end. -handler_id(#{type := Type, name := Name}) -> - binary_to_atom(<<"trace_", (atom_to_binary(Type))/binary, "_", Name/binary>>). +handler_id(Name, Type) -> + TypeBin = atom_to_binary(Type), + case io_lib:printable_unicode_list(binary_to_list(Name)) of + true when byte_size(Name) < 256 -> + NameHash = base64:encode(crypto:hash(sha, Name)), + {ok, binary_to_atom(<<"trace_", TypeBin/binary, "_", NameHash/binary>>)}; + true -> + {error, <<"Name length must < 256">>}; + false -> + {error, <<"Name must printable unicode">>} + end. ensure_bin(List) when is_list(List) -> iolist_to_binary(List); ensure_bin(Bin) when is_binary(Bin) -> Bin. @@ -203,4 +217,4 @@ ensure_list(List) when is_list(List) -> List. show_prompts(ok, Who, Msg) -> ?LOG(info, Msg ++ " ~p " ++ "successfully~n", [Who]); show_prompts({error, Reason}, Who, Msg) -> - ?LOG(error, Msg ++ " ~p " ++ "failed by ~p~n", [Who, Reason]). + ?LOG(error, Msg ++ " ~p " ++ "failed with ~p~n", [Who, Reason]). diff --git a/test/emqx_trace_handler_SUITE.erl b/test/emqx_trace_handler_SUITE.erl index 5e909c45b..e314c7994 100644 --- a/test/emqx_trace_handler_SUITE.erl +++ b/test/emqx_trace_handler_SUITE.erl @@ -70,12 +70,12 @@ t_trace_clientid(_Config) -> ?assert(filelib:is_regular("tmp/client3.log")), %% Get current traces - ?assertEqual([#{type => clientid, filter => "client", name => <<"client">>, - id => trace_clientid_client, level => debug, dst => "tmp/client.log"}, - #{type => clientid, filter => "client2", name => <<"client2">>, - id => trace_clientid_client2, level => debug, dst => "tmp/client2.log"}, - #{type => clientid, filter => "client3", name => <<"client3">>, - id => trace_clientid_client3, level => debug, dst => "tmp/client3.log"} + ?assertMatch([#{type := clientid, filter := "client", name := <<"client">>, + level := debug, dst := "tmp/client.log"}, + #{type := clientid, filter := "client2", name := <<"client2">> + , level := debug, dst := "tmp/client2.log"}, + #{type := clientid, filter := "client3", name := <<"client3">>, + level := debug, dst := "tmp/client3.log"} ], emqx_trace_handler:running()), %% Client with clientid = "client" publishes a "hi" message to "a/b/c". @@ -116,10 +116,10 @@ t_trace_topic(_Config) -> ?assert(filelib:is_regular("tmp/topic_trace_y.log")), %% Get current traces - ?assertEqual([#{type => topic, filter => <<"x/#">>, id => 'trace_topic_x/#', - level => debug, dst => "tmp/topic_trace_x.log", name => <<"x/#">>}, - #{type => topic, filter => <<"y/#">>, id => 'trace_topic_y/#', - name => <<"y/#">>, level => debug, dst => "tmp/topic_trace_y.log"} + ?assertMatch([#{type := topic, filter := <<"x/#">>, + level := debug, dst := "tmp/topic_trace_x.log", name := <<"x/#">>}, + #{type := topic, filter := <<"y/#">>, + name := <<"y/#">>, level := debug, dst := "tmp/topic_trace_y.log"} ], emqx_trace_handler:running()), @@ -159,12 +159,12 @@ t_trace_ip_address(_Config) -> ?assert(filelib:is_regular("tmp/ip_trace_y.log")), %% Get current traces - ?assertEqual([#{type => ip_address, filter => "127.0.0.1", - id => 'trace_ip_address_127.0.0.1', name => <<"127.0.0.1">>, - level => debug, dst => "tmp/ip_trace_x.log"}, - #{type => ip_address, filter => "192.168.1.1", - id => 'trace_ip_address_192.168.1.1', name => <<"192.168.1.1">>, - level => debug, dst => "tmp/ip_trace_y.log"} + ?assertMatch([#{type := ip_address, filter := "127.0.0.1", + name := <<"127.0.0.1">>, + level := debug, dst := "tmp/ip_trace_x.log"}, + #{type := ip_address, filter := "192.168.1.1", + name := <<"192.168.1.1">>, + level := debug, dst := "tmp/ip_trace_y.log"} ], emqx_trace_handler:running()), From 0f5282487287715e2c7d3397d6413a27ae15724d Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 22 Nov 2021 16:49:58 +0100 Subject: [PATCH 089/104] refactor(trace): hash non-printable or too long names --- src/emqx_trace_handler.erl | 42 ++++++++++++++--------------- test/emqx_trace_handler_tests.erl | 44 +++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 21 deletions(-) create mode 100644 test/emqx_trace_handler_tests.erl diff --git a/src/emqx_trace_handler.erl b/src/emqx_trace_handler.erl index 7006fc815..328d734f0 100644 --- a/src/emqx_trace_handler.erl +++ b/src/emqx_trace_handler.erl @@ -160,22 +160,16 @@ filter_ip_address(#{meta := #{peername := Peername}} = Log, {IP, _Name}) -> filter_ip_address(_Log, _ExpectId) -> ignore. install_handler(Who = #{name := Name, type := Type}, Level, LogFile) -> - case handler_id(Name, Type) of - {ok, HandlerId} -> - Config = #{ - level => Level, + HandlerId = handler_id(Name, Type), + Config = #{ level => Level, formatter => ?FORMAT, filter_default => stop, filters => filters(Who), config => ?CONFIG(LogFile) - }, - Res = logger:add_handler(HandlerId, logger_disk_log_h, Config), - show_prompts(Res, Who, "Start trace"), - Res; - {error, _Reason} = Error -> - show_prompts(Error, Who, "Start trace"), - Error - end. + }, + Res = logger:add_handler(HandlerId, logger_disk_log_h, Config), + show_prompts(Res, Who, "Start trace"), + Res. filters(#{type := clientid, filter := Filter, name := Name}) -> [{clientid, {fun ?MODULE:filter_clientid/2, {ensure_list(Filter), Name}}}]; @@ -197,17 +191,23 @@ filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) end. handler_id(Name, Type) -> - TypeBin = atom_to_binary(Type), - case io_lib:printable_unicode_list(binary_to_list(Name)) of - true when byte_size(Name) < 256 -> - NameHash = base64:encode(crypto:hash(sha, Name)), - {ok, binary_to_atom(<<"trace_", TypeBin/binary, "_", NameHash/binary>>)}; - true -> - {error, <<"Name length must < 256">>}; - false -> - {error, <<"Name must printable unicode">>} + try + do_handler_id(Name, Type) + catch + _ : _ -> + Hash = emqx_misc:bin2hexstr_a_f_lower(crypto:hash(md5, Name)), + do_handler_id(Hash, Type) end. +%% Handler ID must be an atom. +do_handler_id(Name, Type) -> + TypeStr = atom_to_list(Type), + NameStr = unicode:characters_to_list(Name, utf8), + FullNameStr = "trace_" ++ TypeStr ++ "_" ++ NameStr, + true = io_lib:printable_unicode_list(FullNameStr), + FullNameBin = unicode:characters_to_binary(FullNameStr, utf8), + binary_to_atom(FullNameBin, utf8). + ensure_bin(List) when is_list(List) -> iolist_to_binary(List); ensure_bin(Bin) when is_binary(Bin) -> Bin. diff --git a/test/emqx_trace_handler_tests.erl b/test/emqx_trace_handler_tests.erl new file mode 100644 index 000000000..40586810e --- /dev/null +++ b/test/emqx_trace_handler_tests.erl @@ -0,0 +1,44 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_trace_handler_tests). + +-include_lib("eunit/include/eunit.hrl"). + +handler_id_test_() -> + [{"normal_printable", + fun() -> + ?assertEqual(trace_topic_t1, emqx_trace_handler:handler_id("t1", topic)) + end}, + {"normal_unicode", + fun() -> + ?assertEqual('trace_topic_äø»é¢˜', emqx_trace_handler:handler_id("äø»é¢˜", topic)) + end + }, + {"not_printable", + fun() -> + ?assertEqual('trace_topic_93b885adfe0da089cdf634904fd59f71', + emqx_trace_handler:handler_id("\0", topic)) + end + }, + {"too_long", + fun() -> + T = lists:duplicate(250, $a), + ?assertEqual('trace_topic_1bdbdf1c9087c796394bcda5789f7206', + emqx_trace_handler:handler_id(T, topic)) + end + } + ]. From 2514f474b0290a3417758b4e16442935bef56c21 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 23 Nov 2021 02:33:20 +0100 Subject: [PATCH 090/104] Sync v4.3 to v4.4 (#6262) * fix(http): fix duplicate http headers * chore(appup): add appup.src * fix(appup): fix multiply defined module in appup * chore(appup): fix wrong version * chore(ekka): Bump version to 0.8.1.5 * fix(update_appup): Fix warnings, add support for external repos * build: use find command's -delete option * ci: do not sync master branch * build: ensure openssl11 * build: copy only libcrypto and libtinfo * fix(trace): handler_id now always return atom Co-authored-by: zhouzb Co-authored-by: k32 <10274441+k32@users.noreply.github.com> --- .ci/build_packages/tests.sh | 6 ++ .github/workflows/git_sync.yaml | 7 +-- Makefile | 2 +- .../emqx_auth_http/src/emqx_auth_http.app.src | 2 +- .../src/emqx_auth_http.appup.src | 10 ++++ .../emqx_auth_http/src/emqx_auth_http_app.erl | 2 +- .../emqx_auth_http/src/emqx_auth_http_cli.erl | 2 +- .../src/emqx_trace/emqx_trace.erl | 18 +++--- apps/emqx_web_hook/src/emqx_web_hook.app.src | 2 +- .../emqx_web_hook/src/emqx_web_hook.appup.src | 12 +++- .../src/emqx_web_hook_actions.erl | 11 ++-- apps/emqx_web_hook/src/emqx_web_hook_app.erl | 4 +- bin/emqx | 7 ++- build | 6 +- deploy/packages/rpm/emqx.spec | 6 ++ scripts/update_appup.escript | 40 +++++-------- src/emqx.app.src | 2 +- src/emqx.appup.src | 56 +++++++++++++------ src/emqx_http_lib.erl | 4 +- src/emqx_trace_handler.erl | 6 +- test/emqx_http_lib_tests.erl | 2 +- 21 files changed, 117 insertions(+), 90 deletions(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 68d8f2df2..4211d1d8d 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -88,6 +88,12 @@ emqx_test(){ ;; "rpm") packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm) + + if [[ "${ARCH}" == "amd64" && $(rpm -E '%{rhel}') == 7 ]] ; then + # EMQX OTP requires openssl11 to have TLS1.3 support + yum install -y openssl11 + fi + rpm -ivh "${PACKAGE_PATH}/${packagename}" if ! rpm -q emqx | grep -q emqx; then echo "package install error" diff --git a/.github/workflows/git_sync.yaml b/.github/workflows/git_sync.yaml index 50fa8c364..2941f5173 100644 --- a/.github/workflows/git_sync.yaml +++ b/.github/workflows/git_sync.yaml @@ -3,7 +3,6 @@ name: Sync to enterprise on: push: branches: - - master - main-v* jobs: @@ -23,11 +22,7 @@ jobs: id: create_pull_request run: | set -euo pipefail - if [ "$GITHUB_REF" = "refs/heads/master" ]; then - EE_REF="refs/heads/enterprise" - else - EE_REF="${GITHUB_REF}-enterprise" - fi + EE_REF="${GITHUB_REF}-enterprise" R=$(curl --silent --show-error \ -H "Accept: application/vnd.github.v3+json" \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ diff --git a/Makefile b/Makefile index 34f41c34c..9a0d55511 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ $(PROFILES:%=clean-%): rm rebar.lock \ rm -rf _build/$(@:clean-%=%)/rel; \ find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ - find _build/$(@:clean-%=%) -type l | xargs rm -i -f ; \ + find _build/$(@:clean-%=%) -type l -delete; \ fi .PHONY: clean-all diff --git a/apps/emqx_auth_http/src/emqx_auth_http.app.src b/apps/emqx_auth_http/src/emqx_auth_http.app.src index 51ddcc69b..c4695348b 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.app.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_http, [{description, "EMQ X Authentication/ACL with HTTP API"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_http_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_auth_http/src/emqx_auth_http.appup.src b/apps/emqx_auth_http/src/emqx_auth_http.appup.src index d1b1061d4..8ebc195dd 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.appup.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.appup.src @@ -2,12 +2,22 @@ {VSN, [ + {"4.3.2", [ + {apply, {application, stop, [emqx_auth_http]}}, + {load_module, emqx_auth_http_app, brutal_purge, soft_purge,[]}, + {load_module, emqx_auth_http_cli, brutal_purge, soft_purge,[]} + ]}, {<<"4.3.[0-1]">>, [ {restart_application, emqx_auth_http} ]}, {<<".*">>, []} ], [ + {"4.3.2", [ + {apply, {application, stop, [emqx_auth_http]}}, + {load_module, emqx_auth_http_app, brutal_purge, soft_purge,[]}, + {load_module, emqx_auth_http_cli, brutal_purge, soft_purge,[]} + ]}, {<<"4.3.[0-1]">>, [ {restart_application, emqx_auth_http} ]}, diff --git a/apps/emqx_auth_http/src/emqx_auth_http_app.erl b/apps/emqx_auth_http/src/emqx_auth_http_app.erl index 2958b93ea..20f2c381e 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_app.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_app.erl @@ -150,7 +150,7 @@ ensure_content_type_header(Method, Headers) when Method =:= post orelse Method =:= put -> Headers; ensure_content_type_header(_Method, Headers) -> - lists:keydelete("content-type", 1, Headers). + lists:keydelete(<<"content-type">>, 1, Headers). path(#{path := "", 'query' := Query}) -> "?" ++ Query; diff --git a/apps/emqx_auth_http/src/emqx_auth_http_cli.erl b/apps/emqx_auth_http/src/emqx_auth_http_cli.erl index 5ef222ae5..d01f571e5 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_cli.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_cli.erl @@ -32,7 +32,7 @@ request(PoolName, get, Path, Headers, Params, Timeout) -> reply(ehttpc:request(PoolName, get, {NewPath, Headers}, Timeout)); request(PoolName, post, Path, Headers, Params, Timeout) -> - Body = case proplists:get_value("content-type", Headers) of + Body = case proplists:get_value(<<"content-type">>, Headers) of "application/x-www-form-urlencoded" -> cow_qs:qs(bin_kw(Params)); "application/json" -> diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index 2190e5bab..c71a457a6 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -379,16 +379,12 @@ to_trace(TraceParam) -> {error, "type=[topic,clientid,ip_address] required"}; {ok, #?TRACE{filter = undefined}} -> {error, "topic/clientid/ip_address filter required"}; - {ok, TraceRec0 = #?TRACE{name = Name, type = Type}} -> - case emqx_trace_handler:handler_id(Name, Type) of - {ok, _} -> - case fill_default(TraceRec0) of - #?TRACE{start_at = Start, end_at = End} when End =< Start -> - {error, "failed by start_at >= end_at"}; - TraceRec -> {ok, TraceRec} - end; - {error, Reason} -> - {error, Reason} + {ok, TraceRec0 = #?TRACE{}} -> + case fill_default(TraceRec0) of + #?TRACE{start_at = Start, end_at = End} when End =< Start -> + {error, "failed by start_at >= end_at"}; + TraceRec -> + {ok, TraceRec} end end. @@ -408,7 +404,7 @@ fill_default(Trace) -> Trace. to_trace([], Rec) -> {ok, Rec}; to_trace([{name, Name} | Trace], Rec) -> - case io_lib:printable_unicode_list(binary_to_list(Name)) of + case io_lib:printable_unicode_list(unicode:characters_to_list(Name, utf8)) of true -> case binary:match(Name, [<<"/">>], []) of nomatch -> to_trace(Trace, Rec#?TRACE{name = Name}); diff --git a/apps/emqx_web_hook/src/emqx_web_hook.app.src b/apps/emqx_web_hook/src/emqx_web_hook.app.src index e0632625f..0a99a2e0a 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.app.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.app.src @@ -1,6 +1,6 @@ {application, emqx_web_hook, [{description, "EMQ X WebHook Plugin"}, - {vsn, "4.3.7"}, % strict semver, bump manually! + {vsn, "4.3.8"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_web_hook_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index ec716f45c..40bdfbdbf 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -1,6 +1,10 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.5",[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + [{"4.3.7", + [{apply,{application,stop,[emqx_web_hook]}}, + {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + {"4.3.5",[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<"4.3.[0-2]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, @@ -9,7 +13,11 @@ {<<"4.3.[3-4]">>, [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.5",[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + [{"4.3.7", + [{apply,{application,stop,[emqx_web_hook]}}, + {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + {"4.3.5",[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<"4.3.[0-2]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index 79aefdb85..68e3e85d4 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -295,7 +295,7 @@ create_req(_, Path, Headers, Body) -> parse_action_params(Params = #{<<"url">> := URL}) -> {ok, #{path := CommonPath}} = emqx_http_lib:uri_parse(URL), Method = method(maps:get(<<"method">>, Params, <<"POST">>)), - Headers = headers(maps:get(<<"headers">>, Params, undefined)), + Headers = headers(maps:get(<<"headers">>, Params, #{})), NHeaders = ensure_content_type_header(Headers, Method), #{method => Method, path => merge_path(CommonPath, maps:get(<<"path">>, Params, <<>>)), @@ -307,7 +307,7 @@ parse_action_params(Params = #{<<"url">> := URL}) -> ensure_content_type_header(Headers, Method) when Method =:= post orelse Method =:= put -> Headers; ensure_content_type_header(Headers, _Method) -> - lists:keydelete("content-type", 1, Headers). + lists:keydelete(<<"content-type">>, 1, Headers). merge_path(CommonPath, <<>>) -> l2b(CommonPath); @@ -326,11 +326,8 @@ method(POST) when POST == <<"POST">>; POST == <<"post">> -> post; method(PUT) when PUT == <<"PUT">>; PUT == <<"put">> -> put; method(DEL) when DEL == <<"DELETE">>; DEL == <<"delete">> -> delete. -headers(undefined) -> []; -headers(Headers) when is_map(Headers) -> - headers(maps:to_list(Headers)); -headers(Headers) when is_list(Headers) -> - [{string:to_lower(str(K)), str(V)} || {K, V} <- Headers]. +headers(Headers) -> + emqx_http_lib:normalise_headers(maps:to_list(Headers)). str(Str) when is_list(Str) -> Str; str(Atom) when is_atom(Atom) -> atom_to_list(Atom); diff --git a/apps/emqx_web_hook/src/emqx_web_hook_app.erl b/apps/emqx_web_hook/src/emqx_web_hook_app.erl index 8265fa664..e728aca9d 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_app.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_app.erl @@ -87,7 +87,7 @@ translate_env() -> application:set_env(?APP, path, Path), application:set_env(?APP, pool_opts, PoolOpts), Headers = application:get_env(?APP, headers, []), - NHeaders = set_content_type(Headers), + NHeaders = set_content_type(emqx_http_lib:normalise_headers(Headers)), application:set_env(?APP, headers, NHeaders). path(#{path := "", 'query' := Query}) -> @@ -100,5 +100,5 @@ path(#{path := Path}) -> Path. set_content_type(Headers) -> - NHeaders = proplists:delete(<<"Content-Type">>, proplists:delete(<<"content-type">>, Headers)), + NHeaders = proplists:delete(<<"content-type">>, Headers), [{<<"content-type">>, <<"application/json">>} | NHeaders]. diff --git a/bin/emqx b/bin/emqx index e68c1f8c2..6a80f4d8d 100755 --- a/bin/emqx +++ b/bin/emqx @@ -41,9 +41,10 @@ if ! check_eralng_start >/dev/null 2>&1; then export LD_LIBRARY_PATH="$DYNLIBS_DIR:$LD_LIBRARY_PATH" if ! check_eralng_start; then ## it's hopeless - echoerr "FATAL: Unable to start Erlang (with libcrypto)." - echoerr "Please make sure it's running on the correct platform with all required dependencies." - echoerr "This EMQ X release is built for $BUILT_ON" + echoerr "FATAL: Unable to start Erlang." + echoerr "Please make sure openssl-1.1.1 (libcrypto) and libncurses are installed." + echoerr "Also ensure it's running on the correct platform," + echoerr "this EMQ X release is built for $BUILT_ON" exit 1 fi echoerr "WARNING: There seem to be missing dynamic libs from the OS. Using libs from ${DYNLIBS_DIR}" diff --git a/build b/build index 50997f76c..7cad5296b 100755 --- a/build +++ b/build @@ -96,7 +96,11 @@ cp_dyn_libs() { mkdir -p "$target_dir" while read -r so_file; do cp -L "$so_file" "$target_dir/" - done < <(find "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 | xargs -0 ldd | grep -E '^\s+.*=>\s(/lib|/usr)' | awk '{print $3}') + done < <(find "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \ + | xargs -0 ldd \ + | grep -E '(libcrypto)|(libtinfo)' \ + | awk '{print $3}' \ + | sort -u) } ## make_zip turns .tar.gz into a .zip with a slightly different name. diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec index ef63b936c..c6eb56a6f 100644 --- a/deploy/packages/rpm/emqx.spec +++ b/deploy/packages/rpm/emqx.spec @@ -19,6 +19,12 @@ BuildRoot: %{_tmppath}/%{_name}-%{_version}-root Provides: %{_name} AutoReq: 0 +%if "%{_arch} %{?rhel}" == "amd64 7" +Requires: openssl11 libatomic +%else +Requires: libatomic +%endif + %description EMQX, a distributed, massively scalable, highly extensible MQTT message broker written in Erlang/OTP. diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 8eb1c7967..39a71b492 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -163,18 +163,6 @@ download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) - find_upstream_repo(Remote) -> string:trim(os:cmd("git remote get-url " ++ Remote)). -find_prev_tag(CurrentRelease) -> - case getopt(prev_tag) of - undefined -> - {Maj, Min, Patch} = parse_semver(CurrentRelease), - case Patch of - 0 -> undefined; - _ -> {ok, semver(Maj, Min, Patch - 1)} - end; - Tag -> - {ok, Tag} - end. - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Appup action creation and updating %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -208,10 +196,21 @@ find_old_appup_actions(App, PrevVersion) -> {Upgrade0, Downgrade0} = case locate(ebin_current, App, ".appup") of {ok, AppupFile} -> + log("Found the previous appup file: ~s~n", [AppupFile]), {_, U, D} = read_appup(AppupFile), {U, D}; undefined -> - {[], []} + %% Fallback to the app.src file, in case the + %% application doesn't have a release (useful for the + %% apps that live outside the EMQX monorepo): + case locate(src, App, ".appup.src") of + {ok, AppupSrcFile} -> + log("Using ~s as a source of previous update actions~n", [AppupSrcFile]), + {_, U, D} = read_appup(AppupSrcFile), + {U, D}; + undefined -> + {[], []} + end end, {ensure_version(PrevVersion, Upgrade0), ensure_version(PrevVersion, Downgrade0)}. @@ -390,17 +389,6 @@ is_valid() -> %% Utility functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -parse_semver(Version) -> - case re(Version, "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.[0-9]+)?$") of - {match, [Maj, Min, Patch|_]} -> - {list_to_integer(Maj), list_to_integer(Min), list_to_integer(Patch)}; - _ -> - error({not_a_semver, Version}) - end. - -semver(Maj, Min, Patch) -> - lists:flatten(io_lib:format("~p.~p.~p", [Maj, Min, Patch])). - %% Locate a file in a specified application locate(ebin_current, App, Suffix) -> ReleaseDir = getopt(beams_dir), @@ -425,6 +413,7 @@ bash(Script) -> bash(Script, []). bash(Script, Env) -> + log("+ ~s~n+ Env: ~p~n", [Script, Env]), case cmd("bash", #{args => ["-c", Script], env => Env}) of 0 -> true; _ -> fail("Failed to run command: ~s", [Script]) @@ -456,9 +445,6 @@ fail(Str, Args) -> log(Str ++ "~n", Args), halt(1). -re(Subject, RE) -> - re:run(Subject, RE, [{capture, all_but_first, list}]). - log(Msg) -> log(Msg, []). diff --git a/src/emqx.app.src b/src/emqx.app.src index a107c7ed4..b0e1664cc 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -1,7 +1,7 @@ {application, emqx, [{id, "emqx"}, {description, "EMQ X"}, - {vsn, "4.3.11"}, % strict semver, bump manually! + {vsn, "4.3.12"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]}, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index d40db1093..653249ccb 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,10 +1,14 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.10", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + [{"4.3.11", + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, + {"4.3.10", + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -14,7 +18,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -24,7 +29,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -36,7 +42,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -49,7 +56,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, @@ -63,7 +71,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, @@ -78,7 +87,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, @@ -157,11 +167,15 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.10", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + [{"4.3.11", + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, + {"4.3.10", + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -171,7 +185,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -181,7 +196,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -193,7 +209,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -206,7 +223,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, @@ -220,7 +238,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, @@ -235,7 +254,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_http_lib.erl b/src/emqx_http_lib.erl index 893c260ee..bf83eaea6 100644 --- a/src/emqx_http_lib.erl +++ b/src/emqx_http_lib.erl @@ -96,11 +96,11 @@ do_parse(URI) -> %% underscores replaced with hyphens %% NOTE: assuming the input Headers list is a proplists, %% that is, when a key is duplicated, list header overrides tail -%% e.g. [{"Content_Type", "applicaiton/binary"}, {"content-type", "applicaiton/json"}] +%% e.g. [{"Content_Type", "applicaiton/binary"}, {<<"content-type">>, "applicaiton/json"}] %% results in: [{"content-type", "applicaiton/binary"}] normalise_headers(Headers0) -> F = fun({K0, V}) -> - K = re:replace(K0, "_", "-", [{return,list}]), + K = re:replace(K0, "_", "-", [{return,binary}]), {string:lowercase(K), V} end, Headers = lists:map(F, Headers0), diff --git a/src/emqx_trace_handler.erl b/src/emqx_trace_handler.erl index 328d734f0..ffe581300 100644 --- a/src/emqx_trace_handler.erl +++ b/src/emqx_trace_handler.erl @@ -113,10 +113,8 @@ install(Who, Level, LogFile) -> -spec uninstall(Type :: clientid | topic | ip_address, Name :: binary() | list()) -> ok | {error, term()}. uninstall(Type, Name) -> - case handler_id(ensure_bin(Name), Type) of - {ok, HandlerId} -> uninstall(HandlerId); - {error, Reason} -> {error, Reason} - end. + HandlerId = handler_id(ensure_bin(Name), Type), + uninstall(HandlerId). -spec uninstall(HandlerId :: atom()) -> ok | {error, term()}. uninstall(HandlerId) -> diff --git a/test/emqx_http_lib_tests.erl b/test/emqx_http_lib_tests.erl index 7bcb7d056..54dd85d94 100644 --- a/test/emqx_http_lib_tests.erl +++ b/test/emqx_http_lib_tests.erl @@ -89,6 +89,6 @@ uri_parse_test_() -> ]. normalise_headers_test() -> - ?assertEqual([{"content-type", "applicaiton/binary"}], + ?assertEqual([{<<"content-type">>, "applicaiton/binary"}], emqx_http_lib:normalise_headers([{"Content_Type", "applicaiton/binary"}, {"content-type", "applicaiton/json"}])). From afd55b31e85cb9c80c022569ebe894193530aba6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 23 Nov 2021 02:34:56 +0100 Subject: [PATCH 091/104] build: define default builder docker image tag (#6245) prior to this change, the OTP_VSN varaible was taken from the docker host's OTP version which may differ from the desired OTP version for the docker builder image. --- Makefile | 3 +++ build | 7 +++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 9a0d55511..4ace92341 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,8 @@ REBAR_VERSION = 3.14.3-emqx-8 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14 +export EMQX_DEFAULT_RUNNER = alpine:3.14 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X @@ -96,6 +98,7 @@ $(PROFILES:%=clean-%): .PHONY: clean-all clean-all: + @rm -f rebar.lock @rm -rf _build .PHONY: deps-all diff --git a/build b/build index 7cad5296b..2fca5bdf2 100755 --- a/build +++ b/build @@ -130,13 +130,12 @@ make_zip() { ## This function builds the default docker image based on alpine:3.14 (by default) make_docker() { - EMQX_RUNNER_IMAGE="${EMQX_RUNNER_IMAGE:-alpine:3.14}" - EMQX_RUNNER_IMAGE_COMPACT="$(echo "$EMQX_RUNNER_IMAGE" | tr -d ':')" - EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-2:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}" + EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}" + EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}" set -x docker build --no-cache --pull \ --build-arg BUILD_FROM="${EMQX_BUILDER}" \ - --build-arg RUN_FROM="${EMQX_RUNNER_IMAGE}" \ + --build-arg RUN_FROM="${EMQX_RUNNER}" \ --build-arg EMQX_NAME="$PROFILE" \ --tag "emqx/$PROFILE:${PKG_VSN}" \ -f "${DOCKERFILE}" . From b87ed0666c7920f4facfa4b785d84042d9cc44a6 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 23 Nov 2021 17:54:17 +0800 Subject: [PATCH 092/104] ci: add new steps for push ecr image when release --- .github/workflows/build_packages.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index b2104e618..ca963da68 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -391,6 +391,20 @@ jobs: EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile context: source + - uses: aws-actions/configure-aws-credentials@v1 + if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Push image to aws ecr + if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' + run: | + version=${GITHUB_REF##*/} + docker pull emqx/emqx:${version#v} + docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v} + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + docker push public.ecr.aws/emqx/emqx:${version#v} delete-artifact: runs-on: ubuntu-20.04 From fef3fc27cb8165943514bea0b2e8e163bb42021e Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 26 Nov 2021 10:42:15 +0800 Subject: [PATCH 093/104] refactor(emqx_slow_subs): refactor use moving average (#6287) * refactor(emqx_slow_subs): refactor use moving average * fix(emqx_slow_subs): change elapsed to latency, and fix some error * fix(emqx_slow_subs): fix emqx_mgmt_api.erl indent * fix(emqx_slow_subs): change api name * fix(emqx_slow_subs): fix and improve some code * fix(emqx_slow_subs): move clienid filed from latency_stats to session --- apps/emqx_management/src/emqx_mgmt_api.erl | 33 +- ...x_st_statistics.hrl => emqx_slow_subs.hrl} | 15 +- .../src/emqx_slow_subs/emqx_slow_subs.erl | 319 +++++++++++++++ .../src/emqx_slow_subs/emqx_slow_subs_api.erl | 57 +++ .../emqx_st_statistics/emqx_st_statistics.erl | 379 ------------------ .../emqx_st_statistics_api.erl | 83 ---- etc/emqx.conf | 41 +- ..._statistics.erl => emqx_mod_slow_subs.erl} | 8 +- lib-ce/emqx_modules/src/emqx_mod_sup.erl | 1 + ...ics_SUITE.erl => emqx_slow_subs_SUITE.erl} | 86 ++-- ...SUITE.erl => emqx_slow_subs_api_SUITE.erl} | 97 ++--- priv/emqx.schema | 49 ++- rebar.config.erl | 2 +- src/emqx_session.erl | 105 +++-- .../emqx_message_latency_stats.erl | 105 +++++ src/emqx_slow_subs/emqx_moving_average.erl | 90 +++++ test/emqx_session_SUITE.erl | 16 +- 17 files changed, 826 insertions(+), 660 deletions(-) rename apps/emqx_plugin_libs/include/{emqx_st_statistics.hrl => emqx_slow_subs.hrl} (68%) create mode 100644 apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl create mode 100644 apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl delete mode 100644 apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl delete mode 100644 apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl rename lib-ce/emqx_modules/src/{emqx_mod_st_statistics.erl => emqx_mod_slow_subs.erl} (90%) rename lib-ce/emqx_modules/test/{emqx_st_statistics_SUITE.erl => emqx_slow_subs_SUITE.erl} (53%) rename lib-ce/emqx_modules/test/{emqx_st_statistics_api_SUITE.erl => emqx_slow_subs_api_SUITE.erl} (59%) create mode 100644 src/emqx_slow_subs/emqx_message_latency_stats.erl create mode 100644 src/emqx_slow_subs/emqx_moving_average.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index e068c5384..d42d2a486 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -49,17 +49,46 @@ paginate(Tables, Params, RowFun) -> query_handle(Table) when is_atom(Table) -> qlc:q([R|| R <- ets:table(Table)]); + +query_handle({Table, Opts}) when is_atom(Table) -> + qlc:q([R|| R <- ets:table(Table, Opts)]); + query_handle([Table]) when is_atom(Table) -> qlc:q([R|| R <- ets:table(Table)]); + +query_handle([{Table, Opts}]) when is_atom(Table) -> + qlc:q([R|| R <- ets:table(Table, Opts)]); + query_handle(Tables) -> - qlc:append([qlc:q([E || E <- ets:table(T)]) || T <- Tables]). + Fold = fun({Table, Opts}, Acc) -> + Handle = qlc:q([R|| R <- ets:table(Table, Opts)]), + [Handle | Acc]; + (Table, Acc) -> + Handle = qlc:q([R|| R <- ets:table(Table)]), + [Handle | Acc] + end, + Handles = lists:foldl(Fold, [], Tables), + qlc:append(lists:reverse(Handles)). count(Table) when is_atom(Table) -> ets:info(Table, size); + +count({Table, _Opts}) when is_atom(Table) -> + ets:info(Table, size); + count([Table]) when is_atom(Table) -> ets:info(Table, size); + +count([{Table, _Opts}]) when is_atom(Table) -> + ets:info(Table, size); + count(Tables) -> - lists:sum([count(T) || T <- Tables]). + Fold = fun({Table, _Opts}, Acc) -> + count(Table) ++ Acc; + (Table, Acc) -> + count(Table) ++ Acc + end, + lists:foldl(Fold, 0, Tables). count(Table, Nodes) -> lists:sum([rpc_call(Node, ets, info, [Table, size], 5000) || Node <- Nodes]). diff --git a/apps/emqx_plugin_libs/include/emqx_st_statistics.hrl b/apps/emqx_plugin_libs/include/emqx_slow_subs.hrl similarity index 68% rename from apps/emqx_plugin_libs/include/emqx_st_statistics.hrl rename to apps/emqx_plugin_libs/include/emqx_slow_subs.hrl index 9184c3194..0b5e3a035 100644 --- a/apps/emqx_plugin_libs/include/emqx_st_statistics.hrl +++ b/apps/emqx_plugin_libs/include/emqx_slow_subs.hrl @@ -14,12 +14,15 @@ %% limitations under the License. %%-------------------------------------------------------------------- --define(LOG_TAB, emqx_st_statistics_log). --define(TOPK_TAB, emqx_st_statistics_topk). +-define(TOPK_TAB, emqx_slow_subs_topk). --record(top_k, { rank :: pos_integer() - , topic :: emqx_types:topic() - , average_count :: number() - , average_elapsed :: number()}). +-define(INDEX(Latency, ClientId), {Latency, ClientId}). + +-record(top_k, { index :: index() + , type :: emqx_message_latency_stats:latency_type() + , last_update_time :: pos_integer() + , extra = [] + }). -type top_k() :: #top_k{}. +-type index() :: ?INDEX(non_neg_integer(), emqx_types:clientid()). diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl new file mode 100644 index 000000000..a50b5eb0e --- /dev/null +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -0,0 +1,319 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_slow_subs). + +-behaviour(gen_server). + +-include_lib("include/emqx.hrl"). +-include_lib("include/logger.hrl"). +-include_lib("emqx_plugin_libs/include/emqx_slow_subs.hrl"). + +-logger_header("[SLOW Subs]"). + +-export([ start_link/1, on_stats_update/2, enable/0 + , disable/0, clear_history/0, init_topk_tab/0 + ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-compile(nowarn_unused_type). + +-type state() :: #{ config := proplist:proplist() + , enable := boolean() + , last_tick_at := pos_integer() + }. + +-type log() :: #{ rank := pos_integer() + , clientid := emqx_types:clientid() + , latency := non_neg_integer() + , type := emqx_message_latency_stats:latency_type() + }. + +-type window_log() :: #{ last_tick_at := pos_integer() + , logs := [log()] + }. + +-type message() :: #message{}. + +-import(proplists, [get_value/2]). + +-type stats_update_args() :: #{ clientid := emqx_types:clientid() + , latency := non_neg_integer() + , type := emqx_message_latency_stats:latency_type() + , last_insert_value := non_neg_integer() + , update_time := timer:time() + }. + +-type stats_update_env() :: #{max_size := pos_integer()}. + +-ifdef(TEST). +-define(EXPIRE_CHECK_INTERVAL, timer:seconds(1)). +-else. +-define(EXPIRE_CHECK_INTERVAL, timer:seconds(10)). +-endif. + +-define(NOW, erlang:system_time(millisecond)). +-define(NOTICE_TOPIC_NAME, "slow_subs"). +-define(DEF_CALL_TIMEOUT, timer:seconds(10)). + +%% erlang term order +%% number < atom < reference < fun < port < pid < tuple < list < bit string + +%% ets ordered_set is ascending by term order + +%%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- +%% @doc Start the st_statistics +-spec(start_link(Env :: list()) -> emqx_types:startlink_ret()). +start_link(Env) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). + +%% XXX NOTE:pay attention to the performance here +-spec on_stats_update(stats_update_args(), stats_update_env()) -> true. +on_stats_update(#{clientid := ClientId, + latency := Latency, + type := Type, + last_insert_value := LIV, + update_time := Ts}, + #{max_size := MaxSize}) -> + + LastIndex = ?INDEX(LIV, ClientId), + Index = ?INDEX(Latency, ClientId), + + %% check whether the client is in the table + case ets:lookup(?TOPK_TAB, LastIndex) of + [#top_k{index = Index}] -> + %% if last value == the new value, return + true; + [_] -> + %% if Latency > minimum value, we should update it + %% if Latency < minimum value, maybe it can replace the minimum value + %% so alwyas update at here + %% do we need check if Latency == minimum ??? + ets:insert(?TOPK_TAB, + #top_k{index = Index, type = Type, last_update_time = Ts}), + ets:delete(?TOPK_TAB, LastIndex); + [] -> + %% try to insert + try_insert_to_topk(MaxSize, Index, Latency, Type, Ts) + end. + +clear_history() -> + gen_server:call(?MODULE, ?FUNCTION_NAME, ?DEF_CALL_TIMEOUT). + +enable() -> + gen_server:call(?MODULE, {enable, true}, ?DEF_CALL_TIMEOUT). + +disable() -> + gen_server:call(?MODULE, {enable, false}, ?DEF_CALL_TIMEOUT). + +init_topk_tab() -> + case ets:whereis(?TOPK_TAB) of + undefined -> + ?TOPK_TAB = ets:new(?TOPK_TAB, + [ ordered_set, public, named_table + , {keypos, #top_k.index}, {write_concurrency, true} + , {read_concurrency, true} + ]); + _ -> + ?TOPK_TAB + end. + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([Conf]) -> + notice_tick(Conf), + expire_tick(Conf), + MaxSize = get_value(top_k_num, Conf), + load(MaxSize), + {ok, #{config => Conf, + last_tick_at => ?NOW, + enable => true}}. + +handle_call({enable, Enable}, _From, + #{config := Cfg, enable := IsEnable} = State) -> + State2 = case Enable of + IsEnable -> + State; + true -> + MaxSize = get_value(max_topk_num, Cfg), + load(MaxSize), + State#{enable := true}; + _ -> + unload(), + State#{enable := false} + end, + {reply, ok, State2}; + +handle_call(clear_history, _, State) -> + ets:delete_all_objects(?TOPK_TAB), + {reply, ok, State}; + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info(expire_tick, #{config := Cfg} = State) -> + expire_tick(Cfg), + Logs = ets:tab2list(?TOPK_TAB), + do_clear(Cfg, Logs), + {noreply, State}; + +handle_info(notice_tick, #{config := Cfg} = State) -> + notice_tick(Cfg), + Logs = ets:tab2list(?TOPK_TAB), + do_notification(Logs, State), + {noreply, State#{last_tick_at := ?NOW}}; + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _) -> + unload(), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +expire_tick(_) -> + erlang:send_after(?EXPIRE_CHECK_INTERVAL, self(), ?FUNCTION_NAME). + +notice_tick(Cfg) -> + case get_value(notice_interval, Cfg) of + 0 -> ok; + Interval -> + erlang:send_after(Interval, self(), ?FUNCTION_NAME), + ok + end. + +-spec do_notification(list(), state()) -> ok. +do_notification([], _) -> + ok; + +do_notification(Logs, #{last_tick_at := LastTickTime, config := Cfg}) -> + start_publish(Logs, LastTickTime, Cfg), + ok. + +start_publish(Logs, TickTime, Cfg) -> + emqx_pool:async_submit({fun do_publish/4, [Logs, erlang:length(Logs), TickTime, Cfg]}). + +do_publish([], _, _, _) -> + ok; + +do_publish(Logs, Rank, TickTime, Cfg) -> + BatchSize = get_value(notice_batch_size, Cfg), + do_publish(Logs, BatchSize, Rank, TickTime, Cfg, []). + +do_publish([Log | T], Size, Rank, TickTime, Cfg, Cache) when Size > 0 -> + Cache2 = [convert_to_notice(Rank, Log) | Cache], + do_publish(T, Size - 1, Rank - 1, TickTime, Cfg, Cache2); + +do_publish(Logs, Size, Rank, TickTime, Cfg, Cache) when Size =:= 0 -> + publish(TickTime, Cfg, Cache), + do_publish(Logs, Rank, TickTime, Cfg); + +do_publish([], _, _Rank, TickTime, Cfg, Cache) -> + publish(TickTime, Cfg, Cache), + ok. + +convert_to_notice(Rank, #top_k{index = ?INDEX(Latency, ClientId), + type = Type, + last_update_time = Ts}) -> + #{rank => Rank, + clientid => ClientId, + latency => Latency, + type => Type, + timestamp => Ts}. + +publish(TickTime, Cfg, Notices) -> + WindowLog = #{last_tick_at => TickTime, + logs => lists:reverse(Notices)}, + Payload = emqx_json:encode(WindowLog), + Msg = #message{ id = emqx_guid:gen() + , qos = get_value(notice_qos, Cfg) + , from = ?MODULE + , topic = emqx_topic:systop(?NOTICE_TOPIC_NAME) + , payload = Payload + , timestamp = ?NOW + }, + _ = emqx_broker:safe_publish(Msg), + ok. + +load(MaxSize) -> + _ = emqx:hook('message.slow_subs_stats', + fun ?MODULE:on_stats_update/2, + [#{max_size => MaxSize}]), + ok. + +unload() -> + emqx:unhook('message.slow_subs_stats', fun ?MODULE:on_stats_update/2). + +do_clear(Cfg, Logs) -> + Now = ?NOW, + Interval = get_value(expire_interval, Cfg), + Each = fun(#top_k{index = Index, last_update_time = Ts}) -> + case Now - Ts >= Interval of + true -> + ets:delete(?TOPK_TAB, Index); + _ -> + true + end + end, + lists:foreach(Each, Logs). + +try_insert_to_topk(MaxSize, Index, Latency, Type, Ts) -> + case ets:info(?TOPK_TAB, size) of + Size when Size < MaxSize -> + %% if the size is under limit, insert it directly + ets:insert(?TOPK_TAB, + #top_k{index = Index, type = Type, last_update_time = Ts}); + _Size -> + %% find the minimum value + ?INDEX(Min, _) = First = + case ets:first(?TOPK_TAB) of + ?INDEX(_, _) = I -> I; + _ -> ?INDEX(Latency - 1, <<>>) + end, + + case Latency =< Min of + true -> true; + _ -> + ets:insert(?TOPK_TAB, + #top_k{index = Index, type = Type, last_update_time = Ts}), + + ets:delete(?TOPK_TAB, First) + end + end. diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl new file mode 100644 index 000000000..a7c9fc050 --- /dev/null +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl @@ -0,0 +1,57 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_slow_subs_api). + +-rest_api(#{name => clear_history, + method => 'DELETE', + path => "/slow_subscriptions", + func => clear_history, + descr => "Clear current data and re count slow topic"}). + +-rest_api(#{name => get_history, + method => 'GET', + path => "/slow_subscriptions", + func => get_history, + descr => "Get slow topics statistics record data"}). + +-export([ clear_history/2 + , get_history/2 + ]). + +-include("include/emqx_slow_subs.hrl"). + +-import(minirest, [return/1]). + +%%-------------------------------------------------------------------- +%% HTTP API +%%-------------------------------------------------------------------- + +clear_history(_Bindings, _Params) -> + ok = emqx_slow_subs:clear_history(), + return(ok). + +get_history(_Bindings, Params) -> + RowFun = fun(#top_k{index = ?INDEX(Latency, ClientId), + type = Type, + last_update_time = Ts}) -> + [{clientid, ClientId}, + {latency, Latency}, + {type, Type}, + {last_update_time, Ts}] + end, + Return = emqx_mgmt_api:paginate({?TOPK_TAB, [{traverse, last_prev}]}, Params, RowFun), + return({ok, Return}). diff --git a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl deleted file mode 100644 index 668cb3926..000000000 --- a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl +++ /dev/null @@ -1,379 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_st_statistics). - --behaviour(gen_server). - --include_lib("include/emqx.hrl"). --include_lib("include/logger.hrl"). --include_lib("emqx_plugin_libs/include/emqx_st_statistics.hrl"). - --logger_header("[SLOW TOPICS]"). - --export([ start_link/1, on_publish_done/3, enable/0 - , disable/0, clear_history/0 - ]). - -%% gen_server callbacks --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , terminate/2 - , code_change/3 - ]). - --compile(nowarn_unused_type). - --type state() :: #{ config := proplist:proplist() - , period := pos_integer() - , last_tick_at := pos_integer() - , counter := counters:counters_ref() - , enable := boolean() - }. - --type log() :: #{ topic := emqx_types:topic() - , count := pos_integer() - , average := float() - }. - --type window_log() :: #{ last_tick_at := pos_integer() - , logs := [log()] - }. - --record(slow_log, { topic :: emqx_types:topic() - , count :: integer() %% 0 will be used in initial value - , elapsed :: integer() - }). - --type message() :: #message{}. - --import(proplists, [get_value/2]). - --define(NOW, erlang:system_time(millisecond)). --define(QUOTA_IDX, 1). - --type slow_log() :: #slow_log{}. --type top_k_map() :: #{emqx_types:topic() => top_k()}. - --type publish_done_env() :: #{ ignore_before_create := boolean() - , threshold := pos_integer() - , counter := counters:counters_ref() - }. - --type publish_done_args() :: #{session_rebirth_time => pos_integer()}. - --ifdef(TEST). --define(TOPK_ACCESS, public). --else. --define(TOPK_ACCESS, protected). --endif. - -%% erlang term order -%% number < atom < reference < fun < port < pid < tuple < list < bit string - -%% ets ordered_set is ascending by term order - -%%-------------------------------------------------------------------- -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- -%% @doc Start the st_statistics --spec(start_link(Env :: list()) -> emqx_types:startlink_ret()). -start_link(Env) -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). - --spec on_publish_done(message(), publish_done_args(), publish_done_env()) -> ok. -on_publish_done(#message{timestamp = Timestamp}, - #{session_rebirth_time := Created}, - #{ignore_before_create := IgnoreBeforeCreate}) - when IgnoreBeforeCreate, Timestamp < Created -> - ok; - -on_publish_done(#message{timestamp = Timestamp} = Msg, - _, - #{threshold := Threshold, counter := Counter}) -> - case ?NOW - Timestamp of - Elapsed when Elapsed > Threshold -> - case get_log_quota(Counter) of - true -> - update_log(Msg, Elapsed); - _ -> - ok - end; - _ -> - ok - end. - -clear_history() -> - gen_server:call(?MODULE, ?FUNCTION_NAME). - -enable() -> - gen_server:call(?MODULE, {enable, true}). - -disable() -> - gen_server:call(?MODULE, {enable, false}). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([Env]) -> - erlang:process_flag(trap_exit, true), - init_log_tab(Env), - init_topk_tab(Env), - notification_tick(Env), - Counter = counters:new(1, [write_concurrency]), - set_log_quota(Env, Counter), - Threshold = get_value(threshold_time, Env), - IgnoreBeforeCreate = get_value(ignore_before_create, Env), - load(IgnoreBeforeCreate, Threshold, Counter), - {ok, #{config => Env, - period => 1, - last_tick_at => ?NOW, - counter => Counter, - enable => true}}. - -handle_call({enable, Enable}, _From, - #{config := Cfg, counter := Counter, enable := IsEnable} = State) -> - State2 = case Enable of - IsEnable -> - State; - true -> - Threshold = get_value(threshold_time, Cfg), - IgnoreBeforeCreate = get_value(ignore_before_create, Cfg), - load(IgnoreBeforeCreate, Threshold, Counter), - State#{enable := true}; - _ -> - unload(), - State#{enable := false} - end, - {reply, ok, State2}; - -handle_call(clear_history, _, State) -> - ets:delete_all_objects(?TOPK_TAB), - {reply, ok, State}; - -handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), - {reply, ignored, State}. - -handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), - {noreply, State}. - -handle_info(notification_tick, #{config := Cfg, period := Period} = State) -> - notification_tick(Cfg), - do_notification(State), - {noreply, State#{last_tick_at := ?NOW, - period := Period + 1}}; - -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, _) -> - unload(), - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- -notification_tick(Env) -> - TimeWindow = get_value(time_window, Env), - erlang:send_after(TimeWindow, self(), ?FUNCTION_NAME). - -init_log_tab(_) -> - ?LOG_TAB = ets:new(?LOG_TAB, [ set, public, named_table - , {keypos, #slow_log.topic}, {write_concurrency, true} - , {read_concurrency, true} - ]). - -init_topk_tab(_) -> - ?TOPK_TAB = ets:new(?TOPK_TAB, [ set, ?TOPK_ACCESS, named_table - , {keypos, #top_k.rank}, {write_concurrency, false} - , {read_concurrency, true} - ]). - --spec get_log_quota(counters:counters_ref()) -> boolean(). -get_log_quota(Counter) -> - case counters:get(Counter, ?QUOTA_IDX) of - Quota when Quota > 0 -> - counters:sub(Counter, ?QUOTA_IDX, 1), - true; - _ -> - false - end. - --spec set_log_quota(proplists:proplist(), counters:counters_ref()) -> ok. -set_log_quota(Cfg, Counter) -> - MaxLogNum = get_value(max_log_num, Cfg), - counters:put(Counter, ?QUOTA_IDX, MaxLogNum). - --spec update_log(message(), pos_integer()) -> ok. -update_log(#message{topic = Topic}, Elapsed) -> - _ = ets:update_counter(?LOG_TAB, - Topic, - [{#slow_log.count, 1}, {#slow_log.elapsed, Elapsed}], - #slow_log{topic = Topic, - count = 0, - elapsed = 0}), - ok. - --spec do_notification(state()) -> true. -do_notification(#{last_tick_at := TickTime, - config := Cfg, - period := Period, - counter := Counter}) -> - Logs = ets:tab2list(?LOG_TAB), - ets:delete_all_objects(?LOG_TAB), - start_publish(Logs, TickTime, Cfg), - set_log_quota(Cfg, Counter), - MaxRecord = get_value(top_k_num, Cfg), - update_topk(Logs, MaxRecord, Period). - --spec update_topk(list(slow_log()), pos_integer(), pos_integer()) -> true. -update_topk(Logs, MaxRecord, Period) -> - TopkMap = get_topk_map(Period), - TopkMap2 = update_topk_map(Logs, Period, TopkMap), - SortFun = fun(A, B) -> - A#top_k.average_count > B#top_k.average_count - end, - TopkL = lists:sort(SortFun, maps:values(TopkMap2)), - TopkL2 = lists:sublist(TopkL, 1, MaxRecord), - update_topk_tab(TopkL2). - --spec update_topk_map(list(slow_log()), pos_integer(), top_k_map()) -> top_k_map(). -update_topk_map([#slow_log{topic = Topic, - count = LogTimes, - elapsed = LogElapsed} | T], Period, TopkMap) -> - case maps:get(Topic, TopkMap, undefined) of - undefined -> - Record = #top_k{rank = 1, - topic = Topic, - average_count = LogTimes, - average_elapsed = LogElapsed}, - TopkMap2 = TopkMap#{Topic => Record}, - update_topk_map(T, Period, TopkMap2); - #top_k{average_count = AvgCount, - average_elapsed = AvgElapsed} = Record -> - NewPeriod = Period + 1, - %% (a + b) / c = a / c + b / c - %% average_count(elapsed) dived NewPeriod in function get_topk_maps - AvgCount2 = AvgCount + LogTimes / NewPeriod, - AvgElapsed2 = AvgElapsed + LogElapsed / NewPeriod, - Record2 = Record#top_k{average_count = AvgCount2, - average_elapsed = AvgElapsed2}, - update_topk_map(T, Period, TopkMap#{Topic := Record2}) - end; - -update_topk_map([], _, TopkMap) -> - TopkMap. - --spec update_topk_tab(list(top_k())) -> true. -update_topk_tab(Records) -> - Zip = fun(Rank, Item) -> Item#top_k{rank = Rank} end, - Len = erlang:length(Records), - RankedTopics = lists:zipwith(Zip, lists:seq(1, Len), Records), - ets:insert(?TOPK_TAB, RankedTopics). - -start_publish(Logs, TickTime, Cfg) -> - emqx_pool:async_submit({fun do_publish/3, [Logs, TickTime, Cfg]}). - -do_publish([], _, _) -> - ok; - -do_publish(Logs, TickTime, Cfg) -> - BatchSize = get_value(notice_batch_size, Cfg), - do_publish(Logs, BatchSize, TickTime, Cfg, []). - -do_publish([Log | T], Size, TickTime, Cfg, Cache) when Size > 0 -> - Cache2 = [convert_to_notice(Log) | Cache], - do_publish(T, Size - 1, TickTime, Cfg, Cache2); - -do_publish(Logs, Size, TickTime, Cfg, Cache) when Size =:= 0 -> - publish(TickTime, Cfg, Cache), - do_publish(Logs, TickTime, Cfg); - -do_publish([], _, TickTime, Cfg, Cache) -> - publish(TickTime, Cfg, Cache), - ok. - -convert_to_notice(#slow_log{topic = Topic, - count = Count, - elapsed = Elapsed}) -> - #{topic => Topic, - count => Count, - average => Elapsed / Count}. - -publish(TickTime, Cfg, Notices) -> - WindowLog = #{last_tick_at => TickTime, - logs => Notices}, - Payload = emqx_json:encode(WindowLog), - _ = emqx:publish(#message{ id = emqx_guid:gen() - , qos = get_value(notice_qos, Cfg) - , from = ?MODULE - , topic = get_topic(Cfg) - , payload = Payload - , timestamp = ?NOW - }), - ok. - -load(IgnoreBeforeCreate, Threshold, Counter) -> - _ = emqx:hook('message.publish_done', - fun ?MODULE:on_publish_done/3, - [#{ignore_before_create => IgnoreBeforeCreate, - threshold => Threshold, - counter => Counter} - ]), - ok. - -unload() -> - emqx:unhook('message.publish_done', fun ?MODULE:on_publish_done/3). - --spec get_topic(proplists:proplist()) -> binary(). -get_topic(Cfg) -> - case get_value(notice_topic, Cfg) of - Topic when is_binary(Topic) -> - Topic; - Topic -> - erlang:list_to_binary(Topic) - end. - --spec get_topk_map(pos_integer()) -> top_k_map(). -get_topk_map(Period) -> - Size = ets:info(?TOPK_TAB, size), - get_topk_map(1, Size, Period, #{}). - --spec get_topk_map(pos_integer(), - non_neg_integer(), pos_integer(), top_k_map()) -> top_k_map(). -get_topk_map(Index, Size, _, TopkMap) when Index > Size -> - TopkMap; -get_topk_map(Index, Size, Period, TopkMap) -> - [#top_k{topic = Topic, - average_count = AvgCount, - average_elapsed = AvgElapsed} = R] = ets:lookup(?TOPK_TAB, Index), - NewPeriod = Period + 1, - TotalTimes = AvgCount * Period, - AvgCount2 = TotalTimes / NewPeriod, - AvgElapsed2 = TotalTimes * AvgElapsed / NewPeriod, - TopkMap2 = TopkMap#{Topic => R#top_k{average_count = AvgCount2, - average_elapsed = AvgElapsed2}}, - get_topk_map(Index + 1, Size, Period, TopkMap2). diff --git a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl deleted file mode 100644 index dc74dcf36..000000000 --- a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl +++ /dev/null @@ -1,83 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_st_statistics_api). - --rest_api(#{name => clear_history, - method => 'DELETE', - path => "/slow_topic", - func => clear_history, - descr => "Clear current data and re count slow topic"}). - --rest_api(#{name => get_history, - method => 'GET', - path => "/slow_topic", - func => get_history, - descr => "Get slow topics statistics record data"}). - --export([ clear_history/2 - , get_history/2 - ]). - --include("include/emqx_st_statistics.hrl"). - --import(minirest, [return/1]). - -%%-------------------------------------------------------------------- -%% HTTP API -%%-------------------------------------------------------------------- - -clear_history(_Bindings, _Params) -> - ok = emqx_st_statistics:clear_history(), - return(ok). - -get_history(_Bindings, Params) -> - PageT = proplists:get_value(<<"_page">>, Params), - LimitT = proplists:get_value(<<"_limit">>, Params), - Page = erlang:binary_to_integer(PageT), - Limit = erlang:binary_to_integer(LimitT), - Start = (Page - 1) * Limit + 1, - Size = ets:info(?TOPK_TAB, size), - End = Start + Limit - 1, - {HasNext, Count, Infos} = get_history(Start, End, Size), - return({ok, #{meta => #{page => Page, - limit => Limit, - hasnext => HasNext, - count => Count}, - data => Infos}}). - - -get_history(Start, _End, Size) when Start > Size -> - {false, 0, []}; - -get_history(Start, End, Size) when End > Size -> - get_history(Start, Size, Size); - -get_history(Start, End, Size) -> - Fold = fun(Rank, Acc) -> - [#top_k{topic = Topic - , average_count = Count - , average_elapsed = Elapsed}] = ets:lookup(?TOPK_TAB, Rank), - - Info = [ {rank, Rank} - , {topic, Topic} - , {count, Count} - , {elapsed, Elapsed}], - - [Info | Acc] - end, - Infos = lists:foldl(Fold, [], lists:seq(Start, End)), - {End < Size, End - Start + 1, Infos}. diff --git a/etc/emqx.conf b/etc/emqx.conf index 78d383930..de5c062a4 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -2217,47 +2217,34 @@ module.presence.qos = 1 ## module.rewrite.sub.rule.1 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2 ##-------------------------------------------------------------------- -## Slow Topic Statistics Module +## Slow Subscribers Statistics Module -## Threshold time of slow topics statistics -## -## Default: 10 seconds -#module.st_statistics.threshold_time = 10S - -## ignore the messages that before than session created -## -## Default: true -#module.st_statistics.ignore_before_create = true - -## Time window of slow topics statistics +## the expire time of the record which in topk ## ## Value: 5 minutes -#module.st_statistics.time_window = 5M +#module.slow_subs.expire_interval = 5m -## Maximum of slow topics log, log will clear when enter new time window +## maximum number of Top-K record ## -## Value: 500 -#module.st_statistics.max_log_num = 500 +## Value: 10 +#module.slow_subs.top_k_num = 10 -## Top-K record for slow topics, update from logs +## enable notification +## publish topk list to $SYS/brokers/${node}/slow_subs per notice_interval +## publish is disabled if set to 0s. ## -## Value: 500 -#module.st_statistics.top_k_num = 500 - -## Topic of notification -## -## Defaut: $slow_topics -#module.st_statistics.notice_topic = $slow_topics +## Defaut: 0s +#module.slow_subs.notice_interval = 0s ## QoS of notification message in notice topic ## ## Defaut: 0 -#module.st_statistics.notice_qos = 0 +#module.slow_subs.notice_qos = 0 ## Maximum information number in one notification ## -## Default: 500 -#module.st_statistics.notice_batch_size = 500 +## Default: 100 +#module.slow_subs.notice_batch_size = 100 ## CONFIG_SECTION_END=modules ================================================== diff --git a/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl b/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl similarity index 90% rename from lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl rename to lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl index d6796122c..b9117fe8b 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl @@ -14,14 +14,14 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mod_st_statistics). +-module(emqx_mod_slow_subs). -behaviour(emqx_gen_mod). -include_lib("include/emqx.hrl"). -include_lib("include/logger.hrl"). --logger_header("[SLOW TOPICS]"). +-logger_header("[SLOW Subs]"). %% emqx_gen_mod callbacks -export([ load/1 @@ -29,7 +29,7 @@ , description/0 ]). --define(LIB, emqx_st_statistics). +-define(LIB, emqx_slow_subs). %%-------------------------------------------------------------------- %% Load/Unload @@ -46,4 +46,4 @@ unload(_Env) -> ok. description() -> - "EMQ X Slow Topic Statistics Module". + "EMQ X Slow Subscribers Statistics Module". diff --git a/lib-ce/emqx_modules/src/emqx_mod_sup.erl b/lib-ce/emqx_modules/src/emqx_mod_sup.erl index 109564f65..4c8f54679 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_sup.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_sup.erl @@ -69,6 +69,7 @@ stop_child(ChildId) -> init([]) -> ok = emqx_tables:new(emqx_modules, [set, public, {write_concurrency, true}]), + emqx_slow_subs:init_topk_tab(), {ok, {{one_for_one, 10, 100}, []}}. %%-------------------------------------------------------------------- diff --git a/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl b/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl similarity index 53% rename from lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl rename to lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl index 8df07f8b8..46f144a57 100644 --- a/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl @@ -14,18 +14,18 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_st_statistics_SUITE). +-module(emqx_slow_subs_SUITE). -compile(export_all). -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). +-include("include/emqx_mqtt.hrl"). -include_lib("include/emqx.hrl"). %-define(LOGT(Format, Args), ct:pal(Format, Args)). --define(LOG_TAB, emqx_st_statistics_log). --define(TOPK_TAB, emqx_st_statistics_topk). +-define(TOPK_TAB, emqx_slow_subs_topk). -define(NOW, erlang:system_time(millisecond)). all() -> emqx_ct:all(?MODULE). @@ -39,11 +39,11 @@ end_per_suite(Config) -> Config. init_per_testcase(_, Config) -> - emqx_mod_st_statistics:load(base_conf()), + emqx_mod_slow_subs:load(base_conf()), Config. end_per_testcase(_, _) -> - emqx_mod_st_statistics:unload(undefined), + emqx_mod_slow_subs:unload([]), ok. %%-------------------------------------------------------------------- @@ -51,62 +51,74 @@ end_per_testcase(_, _) -> %%-------------------------------------------------------------------- t_log_and_pub(_) -> %% Sub topic first - SubBase = "/test", - emqx:subscribe("$slow_topics"), - Clients = start_client(SubBase), + Subs = [{<<"/test1/+">>, ?QOS_1}, {<<"/test2/+">>, ?QOS_2}], + Clients = start_client(Subs), + emqx:subscribe("$SYS/brokers/+/slow_subs"), timer:sleep(1000), Now = ?NOW, %% publish - ?assert(ets:info(?LOG_TAB, size) =:= 0), + lists:foreach(fun(I) -> - Topic = list_to_binary(io_lib:format("~s~p", [SubBase, I])), - Msg = emqx_message:make(Topic, <<"Hello">>), - emqx:publish(Msg#message{timestamp = Now - 1000}) + Topic = list_to_binary(io_lib:format("/test1/~p", [I])), + Msg = emqx_message:make(undefined, ?QOS_1, Topic, <<"Hello">>), + emqx:publish(Msg#message{timestamp = Now - 500}) end, lists:seq(1, 10)), - timer:sleep(2400), + lists:foreach(fun(I) -> + Topic = list_to_binary(io_lib:format("/test2/~p", [I])), + Msg = emqx_message:make(undefined, ?QOS_2, Topic, <<"Hello">>), + emqx:publish(Msg#message{timestamp = Now - 500}) + end, + lists:seq(1, 10)), - ?assert(ets:info(?LOG_TAB, size) =:= 0), - ?assert(ets:info(?TOPK_TAB, size) =:= 3), - try_receive(3), - try_receive(2), + timer:sleep(1000), + Size = ets:info(?TOPK_TAB, size), + %% some time record maybe delete due to it expired + ?assert(Size =< 6 andalso Size >= 4), + + timer:sleep(1500), + Recs = try_receive([]), + RecSum = lists:sum(Recs), + ?assert(RecSum >= 5), + ?assert(lists:all(fun(E) -> E =< 3 end, Recs)), + + timer:sleep(2000), + ?assert(ets:info(?TOPK_TAB, size) =:= 0), [Client ! stop || Client <- Clients], ok. - base_conf() -> - [{top_k_num, 3}, - {threshold_time, 10}, - {notice_qos, 0}, - {notice_batch_size, 3}, - {notice_topic,"$slow_topics"}, - {time_window, 2000}, - {max_log_num, 5}]. + [ {top_k_num, 5} + , {expire_interval, timer:seconds(3)} + , {notice_interval, 1500} + , {notice_qos, 0} + , {notice_batch_size, 3} + ]. -start_client(Base) -> - [spawn(fun() -> - Topic = list_to_binary(io_lib:format("~s~p", [Base, I])), - client(Topic) - end) - || I <- lists:seq(1, 10)]. +start_client(Subs) -> + [spawn(fun() -> client(I, Subs) end) || I <- lists:seq(1, 10)]. -client(Topic) -> +client(I, Subs) -> {ok, C} = emqtt:start_link([{host, "localhost"}, - {clientid, Topic}, + {clientid, io_lib:format("slow_subs_~p", [I])}, {username, <<"plain">>}, {password, <<"plain">>}]), {ok, _} = emqtt:connect(C), - {ok, _, _} = emqtt:subscribe(C, Topic), + + Len = erlang:length(Subs), + Sub = lists:nth(I rem Len + 1, Subs), + _ = emqtt:subscribe(C, Sub), + receive stop -> ok end. -try_receive(L) -> +try_receive(Acc) -> receive {deliver, _, #message{payload = Payload}} -> #{<<"logs">> := Logs} = emqx_json:decode(Payload, [return_maps]), - ?assertEqual(length(Logs), L) + try_receive([length(Logs) | Acc]) after 500 -> - ?assert(false) + Acc end. diff --git a/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl similarity index 59% rename from lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl rename to lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl index 0fb9e904b..2efc7230a 100644 --- a/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl @@ -12,9 +12,9 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- +%%--------------------------------------------------------------------n --module(emqx_st_statistics_api_SUITE). +-module(emqx_slow_subs_api_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -24,7 +24,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx_management/include/emqx_mgmt.hrl"). --include_lib("emqx_plugin_libs/include/emqx_st_statistics.hrl"). +-include_lib("emqx_plugin_libs/include/emqx_slow_subs.hrl"). -define(CONTENT_TYPE, "application/x-www-form-urlencoded"). @@ -33,6 +33,7 @@ -define(API_VERSION, "v4"). -define(BASE_PATH, "api"). +-define(NOW, erlang:system_time(millisecond)). all() -> emqx_ct:all(?MODULE). @@ -48,78 +49,52 @@ end_per_suite(Config) -> Config. init_per_testcase(_, Config) -> - emqx_mod_st_statistics:load(emqx_st_statistics_SUITE:base_conf()), + emqx_mod_slow_subs:load(base_conf()), Config. end_per_testcase(_, Config) -> - emqx_mod_st_statistics:unload(undefined), + emqx_mod_slow_subs:unload([]), Config. +base_conf() -> + [ {top_k_num, 5} + , {expire_interval, timer:seconds(60)} + , {notice_interval, 0} + , {notice_qos, 0} + , {notice_batch_size, 3} + ]. + t_get_history(_) -> - ets:insert(?TOPK_TAB, #top_k{rank = 1, - topic = <<"test">>, - average_count = 12, - average_elapsed = 1500}), + Now = ?NOW, + Each = fun(I) -> + ClientId = erlang:list_to_binary(io_lib:format("test_~p", [I])), + ets:insert(?TOPK_TAB, #top_k{index = ?INDEX(I, ClientId), + type = average, + last_update_time = Now}) + end, - {ok, Data} = request_api(get, api_path(["slow_topic"]), "_page=1&_limit=10", + lists:foreach(Each, lists:seq(1, 5)), + + {ok, Data} = request_api(get, api_path(["slow_subscriptions"]), "_page=1&_limit=10", auth_header_()), + #{meta := Meta, data := [First | _]} = decode(Data), - ShouldRet = #{meta => #{page => 1, - limit => 10, - hasnext => false, - count => 1}, - data => [#{topic => <<"test">>, - rank => 1, - elapsed => 1500, - count => 12}], - code => 0}, + RMeta = #{page => 1, limit => 10, count => 5}, + ?assertEqual(RMeta, Meta), - Ret = decode(Data), + RFirst = #{clientid => <<"test_5">>, + latency => 5, + type => <<"average">>, + last_update_time => Now}, - ?assertEqual(ShouldRet, Ret). - -t_rank_range(_) -> - Insert = fun(Rank) -> - ets:insert(?TOPK_TAB, - #top_k{rank = Rank, - topic = <<"test">>, - average_count = 12, - average_elapsed = 1500}) - end, - lists:foreach(Insert, lists:seq(1, 15)), - - timer:sleep(100), - - {ok, Data} = request_api(get, api_path(["slow_topic"]), "_page=1&_limit=10", - auth_header_()), - - Meta1 = #{page => 1, limit => 10, hasnext => true, count => 10}, - Ret1 = decode(Data), - ?assertEqual(Meta1, maps:get(meta, Ret1)), - - %% End > Size - {ok, Data2} = request_api(get, api_path(["slow_topic"]), "_page=2&_limit=10", - auth_header_()), - - Meta2 = #{page => 2, limit => 10, hasnext => false, count => 5}, - Ret2 = decode(Data2), - ?assertEqual(Meta2, maps:get(meta, Ret2)), - - %% Start > Size - {ok, Data3} = request_api(get, api_path(["slow_topic"]), "_page=3&_limit=10", - auth_header_()), - - Meta3 = #{page => 3, limit => 10, hasnext => false, count => 0}, - Ret3 = decode(Data3), - ?assertEqual(Meta3, maps:get(meta, Ret3)). + ?assertEqual(RFirst, First). t_clear(_) -> - ets:insert(?TOPK_TAB, #top_k{rank = 1, - topic = <<"test">>, - average_count = 12, - average_elapsed = 1500}), + ets:insert(?TOPK_TAB, #top_k{index = ?INDEX(1, <<"test">>), + type = average, + last_update_time = ?NOW}), - {ok, _} = request_api(delete, api_path(["slow_topic"]), [], + {ok, _} = request_api(delete, api_path(["slow_subscriptions"]), [], auth_header_()), ?assertEqual(0, ets:info(?TOPK_TAB, size)). diff --git a/priv/emqx.schema b/priv/emqx.schema index 15f0324cd..61a98f824 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1007,6 +1007,18 @@ end}. {datatype, {duration, s}} ]}. +%% @doc the number of smaples for calculate the average latency of delivery +{mapping, "zone.$name.latency_samples", "emqx.zones", [ + {default, 10}, + {datatype, integer} +]}. + +%% @doc Threshold for slow subscription statistics +{mapping, "zone.$name.latency_stats_threshold", "emqx.zones", [ + {default, "100ms"}, + {datatype, {duration, ms}} +]}. + %% @doc Max Packets that Awaiting PUBREL, 0 means no limit {mapping, "zone.$name.max_awaiting_rel", "emqx.zones", [ {default, 0}, @@ -2218,43 +2230,28 @@ end}. {datatype, string} ]}. -{mapping, "module.st_statistics.threshold_time", "emqx.modules", [ - {default, "10s"}, +{mapping, "module.slow_subs.expire_interval", "emqx.modules", [ + {default, "5m"}, {datatype, {duration, ms}} ]}. -{mapping, "module.st_statistics.ignore_before_create", "emqx.modules", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "module.st_statistics.time_window", "emqx.modules", [ - {default, "5M"}, - {datatype, {duration, ms}} -]}. - -{mapping, "module.st_statistics.max_log_num", "emqx.modules", [ +{mapping, "module.slow_subs.top_k_num", "emqx.modules", [ {default, 500}, {datatype, integer} ]}. -{mapping, "module.st_statistics.top_k_num", "emqx.modules", [ - {default, 500}, - {datatype, integer} +{mapping, "module.slow_subs.notice_interval", "emqx.modules", [ + {default, "0s"}, + {datatype, {duration, ms}} ]}. -{mapping, "module.st_statistics.notice_topic", "emqx.modules", [ - {default, "$slow_topics"}, - {datatype, string} -]}. - -{mapping, "module.st_statistics.notice_qos", "emqx.modules", [ +{mapping, "module.slow_subs.notice_qos", "emqx.modules", [ {default, 0}, {datatype, integer}, {validators, ["range:0-1"]} ]}. -{mapping, "module.st_statistics.notice_batch_size", "emqx.modules", [ +{mapping, "module.slow_subs.notice_batch_size", "emqx.modules", [ {default, 500}, {datatype, integer} ]}. @@ -2283,8 +2280,8 @@ end}. end, TotalRules) end, - SlowTopic = fun() -> - List = cuttlefish_variable:filter_by_prefix("module.st_statistics", Conf), + SlowSubs = fun() -> + List = cuttlefish_variable:filter_by_prefix("module.slow_subs", Conf), [{erlang:list_to_atom(Key), Value} || {[_, _, Key], Value} <- List] end, @@ -2295,7 +2292,7 @@ end}. [{emqx_mod_topic_metrics, []}], [{emqx_mod_delayed, []}], [{emqx_mod_trace, []}], - [{emqx_mod_st_statistics, SlowTopic()}], + [{emqx_mod_slow_subs, SlowSubs()}], [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] ]) end}. diff --git a/rebar.config.erl b/rebar.config.erl index 1000a2c92..6c14b6ae8 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -106,7 +106,7 @@ test_plugins() -> test_deps() -> [ {bbmustache, "1.10.0"} - , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.3.9"}}} + , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.3.11"}}} , meck ]. diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 6122982ae..3e1e48a13 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -95,6 +95,8 @@ -import(emqx_zone, [get_env/3]). -record(session, { + %% Client's id + clientid :: emqx_types:clientid(), %% Clientā€™s Subscriptions. subscriptions :: map(), %% Max subscriptions allowed @@ -121,8 +123,15 @@ %% Awaiting PUBREL Timeout (Unit: millsecond) await_rel_timeout :: timeout(), %% Created at - created_at :: pos_integer() - }). + created_at :: pos_integer(), + %% Message deliver latency stats + latency_stats :: emqx_message_latency_stats:stats() + }). + +%% in the previous code, we will replace the message record with the pubrel atom +%% in the pubrec function, this will lose the creation time of the message, +%% but now we need this time to calculate latency, so now pubrel atom is changed to this record +-record(pubrel_await, {timestamp :: non_neg_integer()}). -type(session() :: #session{}). @@ -148,19 +157,26 @@ mqueue_dropped, next_pkt_id, awaiting_rel_cnt, - awaiting_rel_max + awaiting_rel_max, + latency_stats ]). -define(DEFAULT_BATCH_N, 1000). +-ifdef(TEST). +-define(GET_CLIENT_ID(C), maps:get(clientid, C, <<>>)). +-else. +-define(GET_CLIENT_ID(C), maps:get(clientid, C)). +-endif. %%-------------------------------------------------------------------- %% Init a Session %%-------------------------------------------------------------------- -spec(init(emqx_types:clientinfo(), emqx_types:conninfo()) -> session()). -init(#{zone := Zone}, #{receive_maximum := MaxInflight}) -> - #session{max_subscriptions = get_env(Zone, max_subscriptions, 0), +init(#{zone := Zone} = CInfo, #{receive_maximum := MaxInflight}) -> + #session{clientid = ?GET_CLIENT_ID(CInfo), + max_subscriptions = get_env(Zone, max_subscriptions, 0), subscriptions = #{}, upgrade_qos = get_env(Zone, upgrade_qos, false), inflight = emqx_inflight:new(MaxInflight), @@ -170,7 +186,8 @@ init(#{zone := Zone}, #{receive_maximum := MaxInflight}) -> awaiting_rel = #{}, max_awaiting_rel = get_env(Zone, max_awaiting_rel, 100), await_rel_timeout = timer:seconds(get_env(Zone, await_rel_timeout, 300)), - created_at = erlang:system_time(millisecond) + created_at = erlang:system_time(millisecond), + latency_stats = emqx_message_latency_stats:new(Zone) }. %% @private init mq @@ -227,7 +244,9 @@ info(awaiting_rel_max, #session{max_awaiting_rel = Max}) -> info(await_rel_timeout, #session{await_rel_timeout = Timeout}) -> Timeout div 1000; info(created_at, #session{created_at = CreatedAt}) -> - CreatedAt. + CreatedAt; +info(latency_stats, #session{latency_stats = Stats}) -> + emqx_message_latency_stats:latency(Stats). %% @doc Get stats of the session. -spec(stats(session()) -> emqx_types:stats()). @@ -317,13 +336,12 @@ is_awaiting_full(#session{awaiting_rel = AwaitingRel, -> {ok, emqx_types:message(), session()} | {ok, emqx_types:message(), replies(), session()} | {error, emqx_types:reason_code()}). -puback(PacketId, Session = #session{inflight = Inflight, created_at = CreatedAt}) -> +puback(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> - emqx:run_hook('message.publish_done', - [Msg, #{session_rebirth_time => CreatedAt}]), Inflight1 = emqx_inflight:delete(PacketId, Inflight), - return_with(Msg, dequeue(Session#session{inflight = Inflight1})); + Session2 = update_latency(Msg, Session), + return_with(Msg, dequeue(Session2#session{inflight = Inflight1})); {value, {_Pubrel, _Ts}} -> {error, ?RC_PACKET_IDENTIFIER_IN_USE}; none -> @@ -343,15 +361,13 @@ return_with(Msg, {ok, Publishes, Session}) -> -spec(pubrec(emqx_types:packet_id(), session()) -> {ok, emqx_types:message(), session()} | {error, emqx_types:reason_code()}). -pubrec(PacketId, Session = #session{inflight = Inflight, created_at = CreatedAt}) -> +pubrec(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> - %% execute hook here, because message record will be replaced by pubrel - emqx:run_hook('message.publish_done', - [Msg, #{session_rebirth_time => CreatedAt}]), - Inflight1 = emqx_inflight:update(PacketId, with_ts(pubrel), Inflight), + Update = with_ts(#pubrel_await{timestamp = Msg#message.timestamp}), + Inflight1 = emqx_inflight:update(PacketId, Update, Inflight), {ok, Msg, Session#session{inflight = Inflight1}}; - {value, {pubrel, _Ts}} -> + {value, {_PUBREL, _Ts}} -> {error, ?RC_PACKET_IDENTIFIER_IN_USE}; none -> {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} @@ -380,9 +396,10 @@ pubrel(PacketId, Session = #session{awaiting_rel = AwaitingRel}) -> | {error, emqx_types:reason_code()}). pubcomp(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of - {value, {pubrel, _Ts}} -> + {value, {Pubrel, _Ts}} when is_record(Pubrel, pubrel_await) -> + Session2 = update_latency(Pubrel, Session), Inflight1 = emqx_inflight:delete(PacketId, Inflight), - dequeue(Session#session{inflight = Inflight1}); + dequeue(Session2#session{inflight = Inflight1}); {value, _Other} -> {error, ?RC_PACKET_IDENTIFIER_IN_USE}; none -> @@ -550,11 +567,16 @@ await(PacketId, Msg, Session = #session{inflight = Inflight}) -> %%-------------------------------------------------------------------- -spec(retry(session()) -> {ok, session()} | {ok, replies(), timeout(), session()}). -retry(Session = #session{inflight = Inflight}) -> +retry(Session = #session{inflight = Inflight, retry_interval = RetryInterval}) -> case emqx_inflight:is_empty(Inflight) of true -> {ok, Session}; - false -> retry_delivery(emqx_inflight:to_list(sort_fun(), Inflight), - [], erlang:system_time(millisecond), Session) + false -> + Now = erlang:system_time(millisecond), + Session2 = check_expire_latency(Now, RetryInterval, Session), + retry_delivery(emqx_inflight:to_list(sort_fun(), Inflight), + [], + Now, + Session2) end. retry_delivery([], Acc, _Now, Session = #session{retry_interval = Interval}) -> @@ -581,8 +603,8 @@ retry_delivery(PacketId, Msg, Now, Acc, Inflight) when is_record(Msg, message) - {[{PacketId, Msg1} | Acc], Inflight1} end; -retry_delivery(PacketId, pubrel, Now, Acc, Inflight) -> - Inflight1 = emqx_inflight:update(PacketId, {pubrel, Now}, Inflight), +retry_delivery(PacketId, Pubrel, Now, Acc, Inflight) -> + Inflight1 = emqx_inflight:update(PacketId, {Pubrel, Now}, Inflight), {[{pubrel, PacketId} | Acc], Inflight1}. %%-------------------------------------------------------------------- @@ -626,10 +648,10 @@ resume(ClientInfo = #{clientid := ClientId}, Session = #session{subscriptions = -spec(replay(session()) -> {ok, replies(), session()}). replay(Session = #session{inflight = Inflight}) -> - Pubs = lists:map(fun({PacketId, {pubrel, _Ts}}) -> - {pubrel, PacketId}; + Pubs = lists:map(fun({PacketId, {Pubrel, _Ts}}) when is_record(Pubrel, pubrel_await) -> + {pubrel, PacketId}; ({PacketId, {Msg, _Ts}}) -> - {PacketId, emqx_message:set_flag(dup, true, Msg)} + {PacketId, emqx_message:set_flag(dup, true, Msg)} end, emqx_inflight:to_list(Inflight)), case dequeue(Session) of {ok, NSession} -> {ok, Pubs, NSession}; @@ -677,6 +699,35 @@ next_pkt_id(Session = #session{next_pkt_id = ?MAX_PACKET_ID}) -> next_pkt_id(Session = #session{next_pkt_id = Id}) -> Session#session{next_pkt_id = Id + 1}. +%%-------------------------------------------------------------------- +%% Message Latency Stats +%%-------------------------------------------------------------------- +update_latency(Msg, + #session{clientid = ClientId, + latency_stats = Stats, + created_at = CreateAt} = S) -> + case get_birth_timestamp(Msg, CreateAt) of + 0 -> S; + Ts -> + Latency = erlang:system_time(millisecond) - Ts, + Stats2 = emqx_message_latency_stats:update(ClientId, Latency, Stats), + S#session{latency_stats = Stats2} + end. + +check_expire_latency(Now, Interval, + #session{clientid = ClientId, latency_stats = Stats} = S) -> + Stats2 = emqx_message_latency_stats:check_expire(ClientId, Now, Interval, Stats), + S#session{latency_stats = Stats2}. + +get_birth_timestamp(#message{timestamp = Ts}, CreateAt) when CreateAt =< Ts -> + Ts; + +get_birth_timestamp(#pubrel_await{timestamp = Ts}, CreateAt) when CreateAt =< Ts -> + Ts; + +get_birth_timestamp(_, _) -> + 0. + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- diff --git a/src/emqx_slow_subs/emqx_message_latency_stats.erl b/src/emqx_slow_subs/emqx_message_latency_stats.erl new file mode 100644 index 000000000..f9661ab91 --- /dev/null +++ b/src/emqx_slow_subs/emqx_message_latency_stats.erl @@ -0,0 +1,105 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_message_latency_stats). + +%% API +-export([ new/1, new/2, update/3 + , check_expire/4, latency/1]). + +-define(NOW, erlang:system_time(millisecond)). +-define(MINIMUM_INSERT_INTERVAL, 1000). +-define(MINIMUM_THRESHOLD, 100). + +-opaque stats() :: #{ threshold := number() + , ema := emqx_moving_average:ema() + , last_update_time := timestamp() + , last_access_time := timestamp() %% timestamp of last access top-k + , last_insert_value := non_neg_integer() + }. + +-type timestamp() :: non_neg_integer(). +-type timespan() :: number(). + +-type latency_type() :: average + | expire. + +-import(emqx_zone, [get_env/3]). + +-export_type([stats/0, latency_type/0]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec new(emqx_types:zone()) -> stats(). +new(Zone) -> + Samples = get_env(Zone, latency_samples, 1), + Threshold = get_env(Zone, latency_stats_threshold, ?MINIMUM_THRESHOLD), + new(Samples, Threshold). + +-spec new(non_neg_integer(), number()) -> stats(). +new(SamplesT, ThresholdT) -> + Samples = erlang:max(1, SamplesT), + Threshold = erlang:max(?MINIMUM_THRESHOLD, ThresholdT), + #{ ema => emqx_moving_average:new(exponential, #{period => Samples}) + , threshold => Threshold + , last_update_time => 0 + , last_access_time => 0 + , last_insert_value => 0 + }. + +-spec update(emqx_types:clientid(), number(), stats()) -> stats(). +update(ClientId, Val, #{ema := EMA} = Stats) -> + Now = ?NOW, + #{average := Latency} = EMA2 = emqx_moving_average:update(Val, EMA), + Stats2 = call_hook(ClientId, Now, average, Latency, Stats), + Stats2#{ ema := EMA2 + , last_update_time := ?NOW}. + +-spec check_expire(emqx_types:clientid(), timestamp(), timespan(), stats()) -> stats(). +check_expire(_, Now, Interval, #{last_update_time := LUT} = S) + when LUT >= Now - Interval -> + S; + +check_expire(ClientId, Now, _Interval, #{last_update_time := LUT} = S) -> + Latency = Now - LUT, + call_hook(ClientId, Now, expire, Latency, S). + +-spec latency(stats()) -> number(). +latency(#{ema := #{average := Average}}) -> + Average. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +-spec call_hook(emqx_types:clientid(), timestamp(), latency_type(), timespan(), stats()) -> stats(). +call_hook(_, Now, _, _, #{last_access_time := LIT} = S) + when LIT >= Now - ?MINIMUM_INSERT_INTERVAL -> + S; + +call_hook(_, _, _, Latency, #{threshold := Threshold} = S) + when Latency =< Threshold -> + S; + +call_hook(ClientId, Now, Type, Latency, #{last_insert_value := LIV} = Stats) -> + Arg = #{clientid => ClientId, + latency => erlang:floor(Latency), + type => Type, + last_insert_value => LIV, + update_time => Now}, + emqx:run_hook('message.slow_subs_stats', [Arg]), + Stats#{last_insert_value := Latency, + last_access_time := Now}. diff --git a/src/emqx_slow_subs/emqx_moving_average.erl b/src/emqx_slow_subs/emqx_moving_average.erl new file mode 100644 index 000000000..64c73f987 --- /dev/null +++ b/src/emqx_slow_subs/emqx_moving_average.erl @@ -0,0 +1,90 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @see https://en.wikipedia.org/wiki/Moving_average + +-module(emqx_moving_average). + +%% API +-export([new/0, new/1, new/2, update/2]). + +-type type() :: cumulative + | exponential. + +-type ema() :: #{ type := exponential + , average := 0 | float() + , coefficient := float() + }. + +-type cma() :: #{ type := cumulative + , average := 0 | float() + , count := non_neg_integer() + }. + +-type moving_average() :: ema() + | cma(). + +-define(DEF_EMA_ARG, #{period => 10}). +-define(DEF_AVG_TYPE, exponential). + +-export_type([type/0, moving_average/0, ema/0, cma/0]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec new() -> moving_average(). +new() -> + new(?DEF_AVG_TYPE, #{}). + +-spec new(type()) -> moving_average(). +new(Type) -> + new(Type, #{}). + +-spec new(type(), Args :: map()) -> moving_average(). +new(cumulative, _) -> + #{ type => cumulative + , average => 0 + , count => 0 + }; + +new(exponential, Arg) -> + #{period := Period} = maps:merge(?DEF_EMA_ARG, Arg), + #{ type => exponential + , average => 0 + %% coefficient = 2/(N+1) is a common convention, see the wiki link for details + , coefficient => 2 / (Period + 1) + }. + +-spec update(number(), moving_average()) -> moving_average(). + +update(Val, #{average := 0} = Avg) -> + Avg#{average := Val}; + +update(Val, #{ type := cumulative + , average := Average + , count := Count} = CMA) -> + NewCount = Count + 1, + CMA#{average := (Count * Average + Val) / NewCount, + count := NewCount}; + +update(Val, #{ type := exponential + , average := Average + , coefficient := Coefficient} = EMA) -> + EMA#{average := Coefficient * Val + (1 - Coefficient) * Average}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- diff --git a/test/emqx_session_SUITE.erl b/test/emqx_session_SUITE.erl index cb7c10cae..f838d052a 100644 --- a/test/emqx_session_SUITE.erl +++ b/test/emqx_session_SUITE.erl @@ -180,7 +180,8 @@ t_puback_with_dequeue(_) -> ?assertEqual(<<"t2">>, emqx_message:topic(Msg3)). t_puback_error_packet_id_in_use(_) -> - Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()), + Now = ts(millisecond), + Inflight = emqx_inflight:insert(1, {{pubrel_await, Now}, Now}, emqx_inflight:new()), {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:puback(1, session(#{inflight => Inflight})). @@ -192,10 +193,11 @@ t_pubrec(_) -> Inflight = emqx_inflight:insert(2, {Msg, ts(millisecond)}, emqx_inflight:new()), Session = session(#{inflight => Inflight}), {ok, Msg, Session1} = emqx_session:pubrec(2, Session), - ?assertMatch([{pubrel, _}], emqx_inflight:values(emqx_session:info(inflight, Session1))). + ?assertMatch([{{pubrel_await, _}, _}], emqx_inflight:values(emqx_session:info(inflight, Session1))). t_pubrec_packet_id_in_use_error(_) -> - Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()), + Now = ts(millisecond), + Inflight = emqx_inflight:insert(1, {{pubrel_await, Now}, Now}, emqx_inflight:new()), {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:pubrec(1, session(#{inflight => Inflight})). @@ -211,7 +213,8 @@ t_pubrel_error_packetid_not_found(_) -> {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:pubrel(1, session()). t_pubcomp(_) -> - Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()), + Now = ts(millisecond), + Inflight = emqx_inflight:insert(1, {{pubrel_await, Now}, Now}, emqx_inflight:new()), Session = session(#{inflight => Inflight}), {ok, Session1} = emqx_session:pubcomp(1, Session), ?assertEqual(0, emqx_session:info(inflight_cnt, Session1)). @@ -260,7 +263,7 @@ t_deliver_qos0(_) -> t_deliver_qos1(_) -> ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), {ok, Session} = emqx_session:subscribe( - clientinfo(), <<"t1">>, subopts(#{qos => ?QOS_1}), session()), + clientinfo(), <<"t1">>, subopts(#{qos => ?QOS_1}), session()), Delivers = [delivery(?QOS_1, T) || T <- [<<"t1">>, <<"t2">>]], {ok, [{1, Msg1}, {2, Msg2}], Session1} = emqx_session:deliver(Delivers, Session), ?assertEqual(2, emqx_session:info(inflight_cnt, Session1)), @@ -373,7 +376,7 @@ mqueue(Opts) -> session() -> session(#{}). session(InitFields) when is_map(InitFields) -> maps:fold(fun(Field, Value, Session) -> - emqx_session:set_field(Field, Value, Session) + emqx_session:set_field(Field, Value, Session) end, emqx_session:init(#{zone => channel}, #{receive_maximum => 0}), InitFields). @@ -396,4 +399,3 @@ ts(second) -> erlang:system_time(second); ts(millisecond) -> erlang:system_time(millisecond). - From 8dd4d88d5b1a75b63ccf7778a47a78adee2c6e4c Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 26 Nov 2021 14:57:25 +0800 Subject: [PATCH 094/104] fix(emx_slow_updates): fix the error of topk update (#6312) --- apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl | 6 ++++-- src/emqx_slow_subs/emqx_message_latency_stats.erl | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl index a50b5eb0e..e96c2a907 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -106,8 +106,10 @@ on_stats_update(#{clientid := ClientId, %% check whether the client is in the table case ets:lookup(?TOPK_TAB, LastIndex) of [#top_k{index = Index}] -> - %% if last value == the new value, return - true; + %% if last value == the new value, update the type and last_update_time + %% XXX for clients whose latency are stable for a long time, is it possible to reduce updates? + ets:insert(?TOPK_TAB, + #top_k{index = Index, type = Type, last_update_time = Ts}); [_] -> %% if Latency > minimum value, we should update it %% if Latency < minimum value, maybe it can replace the minimum value diff --git a/src/emqx_slow_subs/emqx_message_latency_stats.erl b/src/emqx_slow_subs/emqx_message_latency_stats.erl index f9661ab91..dfeba1c68 100644 --- a/src/emqx_slow_subs/emqx_message_latency_stats.erl +++ b/src/emqx_slow_subs/emqx_message_latency_stats.erl @@ -95,11 +95,12 @@ call_hook(_, _, _, Latency, #{threshold := Threshold} = S) S; call_hook(ClientId, Now, Type, Latency, #{last_insert_value := LIV} = Stats) -> + ToInsert = erlang:floor(Latency), Arg = #{clientid => ClientId, - latency => erlang:floor(Latency), + latency => ToInsert, type => Type, last_insert_value => LIV, update_time => Now}, emqx:run_hook('message.slow_subs_stats', [Arg]), - Stats#{last_insert_value := Latency, + Stats#{last_insert_value := ToInsert, last_access_time := Now}. From 1b14b792865355000f7d4818ce0e4b398b2d124f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 29 Nov 2021 15:18:09 +0800 Subject: [PATCH 095/104] fix: create trace schema at runtime --- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 2 +- .../src/emqx_trace/emqx_trace.erl | 28 ++++++++----------- .../test/emqx_trace_SUITE.erl | 27 ++++++------------ 3 files changed, 22 insertions(+), 35 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 434e96e21..f21a0f66c 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -240,7 +240,7 @@ t_trace_cmd(_) -> logger:set_primary_config(level, error). t_traces_cmd(_) -> - emqx_trace:mnesia(boot), + emqx_trace:create_table(), Count1 = emqx_mgmt_cli:traces(["list"]), ?assertEqual(0, Count1), Error1 = emqx_mgmt_cli:traces(["start", "test-name", "client", "clientid-dev"]), diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index c71a457a6..135a1f53b 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -22,12 +22,6 @@ -logger_header("[Tracer]"). -%% Mnesia bootstrap --export([mnesia/1]). - --boot_mnesia({mnesia, [boot]}). --copy_mnesia({mnesia, [copy]}). - -export([ publish/1 , subscribe/3 , unsubscribe/2 @@ -57,7 +51,9 @@ -define(MAX_SIZE, 30). -ifdef(TEST). --export([log_file/2]). +-export([ log_file/2 + , create_table/0 + ]). -endif. -export_type([ip_address/0]). @@ -72,15 +68,6 @@ , end_at :: integer() | undefined | '_' }). -mnesia(boot) -> - ok = ekka_mnesia:create_table(?TRACE, [ - {type, set}, - {disc_copies, [node()]}, - {record_name, ?TRACE}, - {attributes, record_info(fields, ?TRACE)}]); -mnesia(copy) -> - ok = ekka_mnesia:copy_table(?TRACE, disc_copies). - publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore; publish(#message{from = From, topic = Topic, payload = Payload}) when is_binary(From); is_atom(From) -> @@ -198,6 +185,7 @@ format(Traces) -> end, Traces). init([]) -> + ok = create_table(), erlang:process_flag(trap_exit, true), OriginLogLevel = emqx_logger:get_primary_log_level(), ok = filelib:ensure_dir(trace_dir()), @@ -208,6 +196,14 @@ init([]) -> TRef = update_trace(Traces), {ok, #{timer => TRef, monitors => #{}, primary_log_level => OriginLogLevel}}. +create_table() -> + ok = ekka_mnesia:create_table(?TRACE, [ + {type, set}, + {disc_copies, [node()]}, + {record_name, ?TRACE}, + {attributes, record_info(fields, ?TRACE)}]), + ok = ekka_mnesia:copy_table(?TRACE, disc_copies). + handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), {reply, ok, State}. diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index ffa2bc1fb..c08211e02 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -33,15 +33,22 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - application:load(emqx_plugin_libs), emqx_ct_helpers:start_apps([]), Config. end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]). -t_base_create_delete(_Config) -> +init_per_testcase(_, Config) -> + load(), ok = emqx_trace:clear(), + Config. + +end_per_testcase(_) -> + unload(), + ok. + +t_base_create_delete(_Config) -> Now = erlang:system_time(second), Start = to_rfc3339(Now), End = to_rfc3339(Now + 30 * 60), @@ -84,7 +91,6 @@ t_base_create_delete(_Config) -> ok. t_create_size_max(_Config) -> - emqx_trace:clear(), lists:map(fun(Seq) -> Name = list_to_binary("name" ++ integer_to_list(Seq)), Trace = [{name, Name}, {type, <<"topic">>}, @@ -100,7 +106,6 @@ t_create_size_max(_Config) -> ok. t_create_failed(_Config) -> - ok = emqx_trace:clear(), UnknownField = [{<<"unknown">>, 12}], {error, Reason1} = emqx_trace:create(UnknownField), ?assertEqual(<<"unknown field: {unknown,12}">>, iolist_to_binary(Reason1)), @@ -139,7 +144,6 @@ t_create_failed(_Config) -> ok. t_create_default(_Config) -> - ok = emqx_trace:clear(), {error, "name required"} = emqx_trace:create([]), ok = emqx_trace:create([{<<"name">>, <<"test-name">>}, {<<"type">>, <<"clientid">>}, {<<"clientid">>, <<"good">>}]), @@ -170,7 +174,6 @@ t_create_default(_Config) -> ok. t_update_enable(_Config) -> - ok = emqx_trace:clear(), Name = <<"test-name">>, Now = erlang:system_time(second), End = list_to_binary(calendar:system_time_to_rfc3339(Now + 2)), @@ -192,8 +195,6 @@ t_update_enable(_Config) -> ok. t_load_state(_Config) -> - emqx_trace:clear(), - load(), Now = erlang:system_time(second), Running = [{<<"name">>, <<"Running">>}, {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/1">>}, {<<"start_at">>, to_rfc3339(Now - 1)}, @@ -218,14 +219,11 @@ t_load_state(_Config) -> Enables2 = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces2), ExpectEnables2 = [{<<"Running">>, false}, {<<"Waiting">>, true}], ?assertEqual(ExpectEnables2, lists:sort(Enables2)), - unload(), ok. t_client_event(_Config) -> application:set_env(emqx, allow_anonymous, true), - emqx_trace:clear(), ClientId = <<"client-test">>, - load(), Now = erlang:system_time(second), Start = to_rfc3339(Now), Name = <<"test_client_id_event">>, @@ -252,12 +250,9 @@ t_client_event(_Config) -> ?assert(erlang:byte_size(Bin) > 0), ?assert(erlang:byte_size(Bin) < erlang:byte_size(Bin2)), ?assert(erlang:byte_size(Bin3) > 0), - unload(), ok. t_get_log_filename(_Config) -> - ok = emqx_trace:clear(), - load(), Now = erlang:system_time(second), Start = calendar:system_time_to_rfc3339(Now), End = calendar:system_time_to_rfc3339(Now + 2), @@ -274,7 +269,6 @@ t_get_log_filename(_Config) -> ?assertEqual(ok, element(1, emqx_trace:get_trace_filename(Name))), ct:sleep(3000), ?assertEqual(ok, element(1, emqx_trace:get_trace_filename(Name))), - unload(), ok. t_trace_file(_Config) -> @@ -290,8 +284,6 @@ t_trace_file(_Config) -> ok. t_download_log(_Config) -> - emqx_trace:clear(), - load(), ClientId = <<"client-test">>, Now = erlang:system_time(second), Start = to_rfc3339(Now), @@ -305,7 +297,6 @@ t_download_log(_Config) -> {ok, ZipFile} = emqx_trace_api:download_zip_log(#{name => Name}, []), ?assert(filelib:file_size(ZipFile) > 0), ok = emqtt:disconnect(Client), - unload(), ok. to_rfc3339(Second) -> From 0d218df14d19632eac72aa075d365dbdd978db07 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 29 Nov 2021 23:08:56 +0800 Subject: [PATCH 096/104] fix: replace ct:sleep/1 by filesync/2 --- .../src/emqx_trace/emqx_trace_api.erl | 6 ++- .../test/emqx_trace_SUITE.erl | 11 +++--- test/emqx_trace_handler_SUITE.erl | 37 ++++++++++++++++--- 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index 7d982c00d..8e052ca66 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -107,8 +107,10 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) -> case Res of {ok, Node, Bin} -> ZipName = ZipDir ++ Node ++ "-" ++ TraceLog, - ok = file:write_file(ZipName, Bin), - [Node ++ "-" ++ TraceLog | Acc]; + case file:write_file(ZipName, Bin) of + ok -> [Node ++ "-" ++ TraceLog | Acc]; + _ -> Acc + end; {error, Node, Reason} -> ?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]), Acc diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index c08211e02..94ab71270 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -229,21 +229,22 @@ t_client_event(_Config) -> Name = <<"test_client_id_event">>, ok = emqx_trace:create([{<<"name">>, Name}, {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), - ct:sleep(200), + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), {ok, _} = emqtt:connect(Client), emqtt:ping(Client), ok = emqtt:publish(Client, <<"/test">>, #{}, <<"1">>, [{qos, 0}]), ok = emqtt:publish(Client, <<"/test">>, #{}, <<"2">>, [{qos, 0}]), - ct:sleep(200), + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), ok = emqx_trace:create([{<<"name">>, <<"test_topic">>}, {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/test">>}, {<<"start_at">>, Start}]), - ct:sleep(200), + ok = emqx_trace_handler_SUITE:filesync(<<"test_topic">>, topic), {ok, Bin} = file:read_file(emqx_trace:log_file(Name, Now)), ok = emqtt:publish(Client, <<"/test">>, #{}, <<"3">>, [{qos, 0}]), ok = emqtt:publish(Client, <<"/test">>, #{}, <<"4">>, [{qos, 0}]), ok = emqtt:disconnect(Client), - ct:sleep(200), + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), + ok = emqx_trace_handler_SUITE:filesync(<<"test_topic">>, topic), {ok, Bin2} = file:read_file(emqx_trace:log_file(Name, Now)), {ok, Bin3} = file:read_file(emqx_trace:log_file(<<"test_topic">>, Now)), ct:pal("Bin ~p Bin2 ~p Bin3 ~p", [byte_size(Bin), byte_size(Bin2), byte_size(Bin3)]), @@ -293,7 +294,7 @@ t_download_log(_Config) -> {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), {ok, _} = emqtt:connect(Client), [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], - ct:sleep(100), + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), {ok, ZipFile} = emqx_trace_api:download_zip_log(#{name => Name}, []), ?assert(filelib:file_size(ZipFile) > 0), ok = emqtt:disconnect(Client), diff --git a/test/emqx_trace_handler_SUITE.erl b/test/emqx_trace_handler_SUITE.erl index e314c7994..b42b1a591 100644 --- a/test/emqx_trace_handler_SUITE.erl +++ b/test/emqx_trace_handler_SUITE.erl @@ -62,7 +62,9 @@ t_trace_clientid(_Config) -> emqx_trace_handler:install(clientid, <<"client4">>, bad_level, "tmp/client4.log"), {error, {handler_not_added, {file_error, ".", eisdir}}} = emqx_trace_handler:install(clientid, <<"client5">>, debug, "."), - ct:sleep(100), + ok = filesync(<<"client">>, clientid), + ok = filesync(<<"client2">>, clientid), + ok = filesync(<<"client3">>, clientid), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/client.log")), @@ -83,7 +85,9 @@ t_trace_clientid(_Config) -> emqtt:connect(T), emqtt:publish(T, <<"a/b/c">>, <<"hi">>), emqtt:ping(T), - ct:sleep(200), + ok = filesync(<<"client">>, clientid), + ok = filesync(<<"client2">>, clientid), + ok = filesync(<<"client3">>, clientid), %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log". {ok, Bin} = file:read_file("tmp/client.log"), @@ -109,7 +113,8 @@ t_trace_topic(_Config) -> emqx_logger:set_log_level(debug), ok = emqx_trace_handler:install(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"), ok = emqx_trace_handler:install(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"), - ct:sleep(100), + ok = filesync(<<"x/#">>, topic), + ok = filesync(<<"y/#">>, topic), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/topic_trace_x.log")), @@ -128,7 +133,8 @@ t_trace_topic(_Config) -> emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), emqtt:subscribe(T, <<"x/y/z">>), emqtt:unsubscribe(T, <<"x/y/z">>), - ct:sleep(200), + ok = filesync(<<"x/#">>, topic), + ok = filesync(<<"y/#">>, topic), {ok, Bin} = file:read_file("tmp/topic_trace_x.log"), ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), @@ -152,7 +158,8 @@ t_trace_ip_address(_Config) -> %% Start tracing ok = emqx_trace_handler:install(ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"), ok = emqx_trace_handler:install(ip_address, "192.168.1.1", all, "tmp/ip_trace_y.log"), - ct:sleep(100), + ok = filesync(<<"127.0.0.1">>, ip_address), + ok = filesync(<<"192.168.1.1">>, ip_address), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/ip_trace_x.log")), @@ -173,7 +180,8 @@ t_trace_ip_address(_Config) -> emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), emqtt:subscribe(T, <<"x/y/z">>), emqtt:unsubscribe(T, <<"x/y/z">>), - ct:sleep(200), + ok = filesync(<<"127.0.0.1">>, ip_address), + ok = filesync(<<"192.168.1.1">>, ip_address), {ok, Bin} = file:read_file("tmp/ip_trace_x.log"), ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), @@ -189,3 +197,20 @@ t_trace_ip_address(_Config) -> {error, _Reason} = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.2">>), emqtt:disconnect(T), ?assertEqual([], emqx_trace_handler:running()). + + +filesync(Name, Type) -> + filesync(Name, Type, 3). + +%% sometime the handler process is not started yet. +filesync(_Name, _Type, 0) -> ok; +filesync(Name, Type, Retry) -> + try + Handler = binary_to_atom(<<"trace_", + (atom_to_binary(Type))/binary, "_", Name/binary>>), + ok = logger_disk_log_h:filesync(Handler) + catch E:R -> + ct:pal("Filesync error:~p ~p~n", [{Name, Type, Retry}, {E, R}]), + ct:sleep(100), + filesync(Name, Type, Retry - 1) + end. From 28e76e498c3ab979a6b44c433b4c1f2a1c9b1d3b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 3 Dec 2021 10:24:36 +0800 Subject: [PATCH 097/104] fix: trace len > 1 return 500 --- apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index 8e052ca66..d243235a4 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -38,7 +38,8 @@ list_trace(_, _Params) -> case emqx_trace:list() of [] -> {ok, []}; List0 -> - List = lists:sort(fun(#{start_at := A}, #{start_at := B}) -> A > B end, List0), + List = lists:sort(fun(#{start_at := A}, #{start_at := B}) -> A > B end, + emqx_trace:format(List0)), Nodes = ekka_mnesia:running_nodes(), TraceSize = cluster_call(?MODULE, get_trace_size, [], 30000), AllFileSize = lists:foldl(fun(F, Acc) -> maps:merge(Acc, F) end, #{}, TraceSize), @@ -50,12 +51,12 @@ list_trace(_, _Params) -> LogSize = collect_file_size(Nodes, FileName, AllFileSize), Trace0 = maps:without([enable, filter], Trace), Trace0#{ log_size => LogSize - , Type => Filter + , Type => iolist_to_binary(Filter) , start_at => list_to_binary(calendar:system_time_to_rfc3339(Start)) , end_at => list_to_binary(calendar:system_time_to_rfc3339(End)) , status => status(Enable, Start, End, Now) } - end, emqx_trace:format(List)), + end, List), {ok, Traces} end. From 48f8c735ea93dbf57dfe9534e5231c63c9fb0f67 Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 3 Dec 2021 18:00:22 +0800 Subject: [PATCH 098/104] feat(emqx_slow_subs): add dyanamic threshold --- .../emqx_modules/src/emqx_mod_slow_subs.erl | 2 + .../test/emqx_slow_subs_SUITE.erl | 3 +- .../test/emqx_slow_subs_api_SUITE.erl | 3 +- priv/emqx.schema | 11 ++- .../emqx_message_latency_stats.erl | 70 +++++++++++-------- 5 files changed, 50 insertions(+), 39 deletions(-) diff --git a/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl b/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl index b9117fe8b..1f5ebebf6 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl @@ -37,6 +37,8 @@ -spec(load(list()) -> ok). load(Env) -> + Threshold = proplists:get_value(threshold, Env), + _ = emqx_message_latency_stats:update_threshold(Threshold), emqx_mod_sup:start_child(?LIB, worker, [Env]), ok. diff --git a/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl b/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl index 46f144a57..f2763ae6c 100644 --- a/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl @@ -88,7 +88,8 @@ t_log_and_pub(_) -> [Client ! stop || Client <- Clients], ok. base_conf() -> - [ {top_k_num, 5} + [ {threshold, 500} + , {top_k_num, 5} , {expire_interval, timer:seconds(3)} , {notice_interval, 1500} , {notice_qos, 0} diff --git a/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl index 2efc7230a..c8f6b9cfc 100644 --- a/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl @@ -57,7 +57,8 @@ end_per_testcase(_, Config) -> Config. base_conf() -> - [ {top_k_num, 5} + [ {threshold, 500} + , {top_k_num, 5} , {expire_interval, timer:seconds(60)} , {notice_interval, 0} , {notice_qos, 0} diff --git a/priv/emqx.schema b/priv/emqx.schema index 61a98f824..facd86455 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1013,12 +1013,6 @@ end}. {datatype, integer} ]}. -%% @doc Threshold for slow subscription statistics -{mapping, "zone.$name.latency_stats_threshold", "emqx.zones", [ - {default, "100ms"}, - {datatype, {duration, ms}} -]}. - %% @doc Max Packets that Awaiting PUBREL, 0 means no limit {mapping, "zone.$name.max_awaiting_rel", "emqx.zones", [ {default, 0}, @@ -2230,6 +2224,11 @@ end}. {datatype, string} ]}. +{mapping, "module.slow_subs.threshold", "emqx.modules", [ + {default, "500ms"}, + {datatype, {duration, ms}} +]}. + {mapping, "module.slow_subs.expire_interval", "emqx.modules", [ {default, "5m"}, {datatype, {duration, ms}} diff --git a/src/emqx_slow_subs/emqx_message_latency_stats.erl b/src/emqx_slow_subs/emqx_message_latency_stats.erl index dfeba1c68..54fe04760 100644 --- a/src/emqx_slow_subs/emqx_message_latency_stats.erl +++ b/src/emqx_slow_subs/emqx_message_latency_stats.erl @@ -17,17 +17,18 @@ -module(emqx_message_latency_stats). %% API --export([ new/1, new/2, update/3 - , check_expire/4, latency/1]). +-export([ new/1, update/3, check_expire/4, latency/1]). + +-export([get_threshold/0, update_threshold/1]). -define(NOW, erlang:system_time(millisecond)). -define(MINIMUM_INSERT_INTERVAL, 1000). --define(MINIMUM_THRESHOLD, 100). +-define(MINIMUM_THRESHOLD, 500). +-define(THRESHOLD_KEY, {?MODULE, threshold}). --opaque stats() :: #{ threshold := number() - , ema := emqx_moving_average:ema() +-opaque stats() :: #{ ema := emqx_moving_average:ema() , last_update_time := timestamp() - , last_access_time := timestamp() %% timestamp of last access top-k + , last_access_time := timestamp() %% timestamp of last try to call hook , last_insert_value := non_neg_integer() }. @@ -44,22 +45,19 @@ %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- --spec new(emqx_types:zone()) -> stats(). -new(Zone) -> - Samples = get_env(Zone, latency_samples, 1), - Threshold = get_env(Zone, latency_stats_threshold, ?MINIMUM_THRESHOLD), - new(Samples, Threshold). --spec new(non_neg_integer(), number()) -> stats(). -new(SamplesT, ThresholdT) -> +-spec new(non_neg_integer() | emqx_types:zone()) -> stats(). +new(SamplesT) when is_integer(SamplesT) -> Samples = erlang:max(1, SamplesT), - Threshold = erlang:max(?MINIMUM_THRESHOLD, ThresholdT), #{ ema => emqx_moving_average:new(exponential, #{period => Samples}) - , threshold => Threshold , last_update_time => 0 , last_access_time => 0 , last_insert_value => 0 - }. + }; + +new(Zone) -> + Samples = get_env(Zone, latency_samples, 1), + new(Samples). -spec update(emqx_types:clientid(), number(), stats()) -> stats(). update(ClientId, Val, #{ema := EMA} = Stats) -> @@ -82,25 +80,35 @@ check_expire(ClientId, Now, _Interval, #{last_update_time := LUT} = S) -> latency(#{ema := #{average := Average}}) -> Average. +-spec update_threshold(pos_integer()) -> pos_integer(). +update_threshold(Threshold) -> + Val = erlang:max(Threshold, ?MINIMUM_THRESHOLD), + persistent_term:put(?THRESHOLD_KEY, Val), + Val. + +get_threshold() -> + persistent_term:get(?THRESHOLD_KEY, ?MINIMUM_THRESHOLD). + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- -spec call_hook(emqx_types:clientid(), timestamp(), latency_type(), timespan(), stats()) -> stats(). -call_hook(_, Now, _, _, #{last_access_time := LIT} = S) - when LIT >= Now - ?MINIMUM_INSERT_INTERVAL -> - S; - -call_hook(_, _, _, Latency, #{threshold := Threshold} = S) - when Latency =< Threshold -> +call_hook(_, _, _, Latency, S) + when Latency =< ?MINIMUM_THRESHOLD -> S; call_hook(ClientId, Now, Type, Latency, #{last_insert_value := LIV} = Stats) -> - ToInsert = erlang:floor(Latency), - Arg = #{clientid => ClientId, - latency => ToInsert, - type => Type, - last_insert_value => LIV, - update_time => Now}, - emqx:run_hook('message.slow_subs_stats', [Arg]), - Stats#{last_insert_value := ToInsert, - last_access_time := Now}. + case get_threshold() >= Latency of + true -> + Stats#{last_access_time := Now}; + _ -> + ToInsert = erlang:floor(Latency), + Arg = #{clientid => ClientId, + latency => ToInsert, + type => Type, + last_insert_value => LIV, + update_time => Now}, + emqx:run_hook('message.slow_subs_stats', [Arg]), + Stats#{last_insert_value := ToInsert, + last_access_time := Now} + end. From 3f49e3186cfda9252d7cf34172a23fe48a86fdee Mon Sep 17 00:00:00 2001 From: lafirest Date: Mon, 6 Dec 2021 17:22:36 +0800 Subject: [PATCH 099/104] fix(emqx_slow_subs): add default threshold macro --- src/emqx_slow_subs/emqx_message_latency_stats.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/emqx_slow_subs/emqx_message_latency_stats.erl b/src/emqx_slow_subs/emqx_message_latency_stats.erl index 54fe04760..49f0c8f1e 100644 --- a/src/emqx_slow_subs/emqx_message_latency_stats.erl +++ b/src/emqx_slow_subs/emqx_message_latency_stats.erl @@ -23,7 +23,8 @@ -define(NOW, erlang:system_time(millisecond)). -define(MINIMUM_INSERT_INTERVAL, 1000). --define(MINIMUM_THRESHOLD, 500). +-define(MINIMUM_THRESHOLD, 100). +-define(DEFAULT_THRESHOLD, 500). -define(THRESHOLD_KEY, {?MODULE, threshold}). -opaque stats() :: #{ ema := emqx_moving_average:ema() @@ -87,7 +88,7 @@ update_threshold(Threshold) -> Val. get_threshold() -> - persistent_term:get(?THRESHOLD_KEY, ?MINIMUM_THRESHOLD). + persistent_term:get(?THRESHOLD_KEY, ?DEFAULT_THRESHOLD). %%-------------------------------------------------------------------- %% Internal functions @@ -98,7 +99,7 @@ call_hook(_, _, _, Latency, S) S; call_hook(ClientId, Now, Type, Latency, #{last_insert_value := LIV} = Stats) -> - case get_threshold() >= Latency of + case Latency =< get_threshold() of true -> Stats#{last_access_time := Now}; _ -> From d4dd4a124c5c08e728e6c40b89c3ead2eafef034 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 6 Dec 2021 18:24:47 +0800 Subject: [PATCH 100/104] fix: trace_name format [A-Za-z0-9-_];waiting status if create time to closed" --- .../src/emqx_trace/emqx_trace.erl | 15 +++++++------ .../src/emqx_trace/emqx_trace_api.erl | 21 ++++++++----------- .../test/emqx_trace_SUITE.erl | 2 +- test/emqx_trace_handler_SUITE.erl | 1 + 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index 135a1f53b..252da8c0f 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -398,15 +398,13 @@ fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) -> fill_default(Trace#?TRACE{end_at = StartAt + 10 * 60}); fill_default(Trace) -> Trace. +-define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_]*$"). + to_trace([], Rec) -> {ok, Rec}; to_trace([{name, Name} | Trace], Rec) -> - case io_lib:printable_unicode_list(unicode:characters_to_list(Name, utf8)) of - true -> - case binary:match(Name, [<<"/">>], []) of - nomatch -> to_trace(Trace, Rec#?TRACE{name = Name}); - _ -> {error, "name cannot contain /"} - end; - false -> {error, "name must printable unicode"} + case re:run(Name, ?NAME_RE) of + nomatch -> {error, "Name should be " ?NAME_RE}; + _ -> to_trace(Trace, Rec#?TRACE{name = Name}) end; to_trace([{type, Type} | Trace], Rec) -> case lists:member(Type, [<<"clientid">>, <<"topic">>, <<"ip_address">>]) of @@ -453,7 +451,8 @@ validate_topic(TopicName) -> to_system_second(At) -> try Sec = calendar:rfc3339_to_system_time(binary_to_list(At), [{unit, second}]), - {ok, Sec} + Now = erlang:system_time(second), + {ok, erlang:max(Now, Sec)} catch error: {badmatch, _} -> {error, ["The rfc3339 specification not satisfied: ", At]} end. diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index d243235a4..d2bca542b 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -135,7 +135,7 @@ stream_log_file(#{name := Name}, Params) -> {ok, Node} -> Position = binary_to_integer(Position0), Bytes = binary_to_integer(Bytes0), - case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of + case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of {ok, Bin} -> Meta = #{<<"position">> => Position + byte_size(Bin), <<"bytes">> => Bytes}, {ok, #{meta => Meta, items => Bin}}; @@ -143,7 +143,7 @@ stream_log_file(#{name := Name}, Params) -> Meta = #{<<"position">> => Size, <<"bytes">> => Bytes}, {ok, #{meta => Meta, items => <<"">>}}; {error, Reason} -> - logger:log(error, "read_file_failed by ~p", [{Name, Reason, Position, Bytes}]), + logger:log(error, "read_file_failed by ~p", [{Node, Name, Reason, Position, Bytes}]), {error, Reason}; {badrpc, nodedown} -> {error, "BadRpc node down"} @@ -165,15 +165,12 @@ get_trace_size() -> %% this is an rpc call for stream_log_file/2 read_trace_file(Name, Position, Limit) -> - TraceDir = emqx_trace:trace_dir(), - {ok, AllFiles} = file:list_dir(TraceDir), - TracePrefix = "trace_" ++ binary_to_list(Name) ++ "_", - Filter = fun(FileName) -> nomatch =/= string:prefix(FileName, TracePrefix) end, - case lists:filter(Filter, AllFiles) of - [TraceFile] -> + case emqx_trace:get_trace_filename(Name) of + {error, _} = Error -> Error; + {ok, TraceFile} -> + TraceDir = emqx_trace:trace_dir(), TracePath = filename:join([TraceDir, TraceFile]), - read_file(TracePath, Position, Limit); - [] -> {error, not_found} + read_file(TracePath, Position, Limit) end. read_file(Path, Offset, Bytes) -> @@ -206,8 +203,8 @@ collect_file_size(Nodes, FileName, AllFiles) -> Acc#{Node => Size} end, #{}, Nodes). -%% status(false, _Start, End, Now) when End > Now -> <<"stopped">>; status(false, _Start, _End, _Now) -> <<"stopped">>; -status(true, Start, _End, Now) when Now < Start -> <<"waiting">>; +%% asynchronously create trace, we should wait 1 seconds +status(true, Start, _End, Now) when Now < Start + 2 -> <<"waiting">>; status(true, _Start, End, Now) when Now >= End -> <<"stopped">>; status(true, _Start, _End, _Now) -> <<"running">>. diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index 94ab71270..4f33e5b7f 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -130,7 +130,7 @@ t_create_failed(_Config) -> InvalidPackets4 = [{<<"name">>, <<"/test">>}, {<<"clientid">>, <<"t">>}, {<<"type">>, <<"clientid">>}], {error, Reason9} = emqx_trace:create(InvalidPackets4), - ?assertEqual(<<"name cannot contain /">>, iolist_to_binary(Reason9)), + ?assertEqual(<<"Name should be ^[A-Za-z]+[A-Za-z0-9-_]*$">>, iolist_to_binary(Reason9)), ?assertEqual({error, "type=[topic,clientid,ip_address] required"}, emqx_trace:create([{<<"name">>, <<"test-name">>}, {<<"clientid">>, <<"good">>}])), diff --git a/test/emqx_trace_handler_SUITE.erl b/test/emqx_trace_handler_SUITE.erl index b42b1a591..cbf455d2d 100644 --- a/test/emqx_trace_handler_SUITE.erl +++ b/test/emqx_trace_handler_SUITE.erl @@ -200,6 +200,7 @@ t_trace_ip_address(_Config) -> filesync(Name, Type) -> + ct:sleep(50), filesync(Name, Type, 3). %% sometime the handler process is not started yet. From e7c765aaa3f5276601e3040f78ed2bef0719d0b7 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 7 Dec 2021 14:30:16 +0800 Subject: [PATCH 101/104] chore(helm): support cluster discovery by dns --- .github/workflows/run_fvt_tests.yaml | 53 ++++++++++++++++--- deploy/charts/emqx/templates/StatefulSet.yaml | 13 +---- deploy/charts/emqx/templates/configmap.yaml | 2 +- deploy/charts/emqx/templates/rbac.yaml | 4 +- deploy/charts/emqx/values.yaml | 13 +++++ deploy/docker/docker-entrypoint.sh | 20 ++++--- 6 files changed, 80 insertions(+), 25 deletions(-) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index a16586bd9..772fd4fa2 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -65,6 +65,12 @@ jobs: helm_test: runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + discovery: + - k8s + - dns steps: - uses: actions/checkout@v1 @@ -104,17 +110,18 @@ jobs: sudo chmod 700 get_helm.sh sudo ./get_helm.sh helm version - - name: run emqx on chart - env: - KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" - timeout-minutes: 5 + - name: setup emqx chart run: | sudo docker save ${TARGET}:${EMQX_TAG} -o emqx.tar.gz sudo k3s ctr image import emqx.tar.gz sed -i -r "s/^appVersion: .*$/appVersion: \"${EMQX_TAG}\"/g" deploy/charts/emqx/Chart.yaml sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml - + - name: run emqx on chart + if: matrix.discovery == 'k8s' + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + run: | helm install emqx \ --set image.repository=${TARGET} \ --set image.pullPolicy=Never \ @@ -124,7 +131,29 @@ jobs: --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ deploy/charts/emqx \ --debug - + - name: run emqx on chart + if: matrix.discovery == 'dns' + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + run: | + helm install emqx \ + --set emqxConfig.EMQX_CLUSTER__DISCOVERY="dns" \ + --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \ + --set emqxConfig.EMQX_CLUSTER__DNS__APP="emqx" \ + --set emqxConfig.EMQX_CLUSTER__DNS__TYPE="srv" \ + --set image.repository=${TARGET} \ + --set image.pullPolicy=Never \ + --set emqxAclConfig="" \ + --set image.pullPolicy=Never \ + --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s \ + --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ + deploy/charts/emqx \ + --debug + - name: waiting emqx started + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + timeout-minutes: 5 + run: | while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \ != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do echo "=============================="; @@ -133,6 +162,18 @@ jobs: echo "waiting emqx started"; sleep 10; done + - name: Check ${{ matrix.kind[0]}} cluster + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + timeout-minutes: 10 + run: | + while + nodes="$(kubectl exec -i emqx-0 -- curl --silent --basic -u admin:public -X GET http://localhost:8081/api/v4/brokers | jq '.data|length')"; + [ "$nodes" != "3" ]; + do + echo "waiting emqx cluster scale" + sleep 1 + done - name: get emqx-0 pods log if: failure() env: diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index dcc03b996..7ad870b6d 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -87,7 +87,9 @@ spec: secret: secretName: {{ .Values.emqxLicneseSecretName }} {{- end }} + {{- if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY | default "k8s") "k8s" }} serviceAccountName: {{ include "emqx.fullname" . }} + {{- end }} {{- if .Values.podSecurityContext.enabled }} securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} @@ -134,17 +136,6 @@ spec: envFrom: - configMapRef: name: {{ include "emqx.fullname" . }}-env - env: - - name: EMQX_NAME - value: {{ .Release.Name }} - - name: EMQX_CLUSTER__K8S__APP_NAME - value: {{ .Release.Name }} - - name: EMQX_CLUSTER__DISCOVERY - value: k8s - - name: EMQX_CLUSTER__K8S__SERVICE_NAME - value: {{ include "emqx.fullname" . }}-headless - - name: EMQX_CLUSTER__K8S__NAMESPACE - value: {{ .Release.Namespace }} resources: {{ toYaml .Values.resources | indent 12 }} volumeMounts: diff --git a/deploy/charts/emqx/templates/configmap.yaml b/deploy/charts/emqx/templates/configmap.yaml index c9c4b4770..328df2000 100644 --- a/deploy/charts/emqx/templates/configmap.yaml +++ b/deploy/charts/emqx/templates/configmap.yaml @@ -10,7 +10,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} data: {{- range $index, $value := .Values.emqxConfig}} - {{$index}}: "{{ $value }}" + {{$index}}: "{{ tpl (printf "%v" $value) $ }}" {{- end}} --- diff --git a/deploy/charts/emqx/templates/rbac.yaml b/deploy/charts/emqx/templates/rbac.yaml index 87cd18178..79b431442 100644 --- a/deploy/charts/emqx/templates/rbac.yaml +++ b/deploy/charts/emqx/templates/rbac.yaml @@ -1,3 +1,4 @@ +{{- if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY | default "k8s") "k8s" }} apiVersion: v1 kind: ServiceAccount metadata: @@ -39,4 +40,5 @@ subjects: roleRef: kind: Role name: {{ include "emqx.fullname" . }} - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 62cf779ea..373ba0849 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -50,7 +50,20 @@ initContainers: {} ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx) emqxConfig: + EMQX_NAME: "{{ .Release.Name }}" + + ## Cluster discovery by dns + # EMQX_CLUSTER__DISCOVERY: "dns" + # EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local" + # EMQX_CLUSTER__DNS__APP: "{{ .Release.Name }}" + # EMQX_CLUSTER__DNS__TYPE: "srv" + + ## Cluster discovery by k8s + EMQX_CLUSTER__DISCOVERY: "k8s" + EMQX_CLUSTER__K8S__APP_NAME: "{{ .Release.Name }}" EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443" + EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless" + EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}" ## The address type is used to extract host from k8s service. ## Value: ip | dns | hostname ## Noteļ¼šHostname is only supported after v4.0-rc.2 diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh index 16b6cb077..1abef430b 100755 --- a/deploy/docker/docker-entrypoint.sh +++ b/deploy/docker/docker-entrypoint.sh @@ -28,12 +28,20 @@ if [[ -z "$EMQX_NAME" ]]; then fi if [[ -z "$EMQX_HOST" ]]; then - if [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then - EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"} - EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX" - elif [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then - EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'} - EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX") + if [[ "$EMQX_CLUSTER__DISCOVERY" == "dns" ]] && \ + [[ "$EMQX_CLUSTER__DNS__TYPE" == "srv" ]] && \ + grep -q "$(hostname).$EMQX_CLUSTER__DNS__NAME" /etc/hosts; then + EMQX_HOST="$(hostname).$EMQX_CLUSTER__DNS__NAME" + elif [[ "$EMQX_CLUSTER__DISCOVERY" == "k8s" ]] && \ + [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && \ + [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then + EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"} + EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX" + elif [[ "$EMQX_CLUSTER__DISCOVERY" == "k8s" ]] && \ + [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && \ + [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then + EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'} + EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX") else EMQX_HOST="$LOCAL_IP" fi From be6160f5bd81aa1c4db2c04882c16b2e883286ad Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 7 Dec 2021 20:24:56 +0800 Subject: [PATCH 102/104] fix: trace handler start time not correct --- .../src/emqx_trace/emqx_trace.erl | 13 +++--- .../src/emqx_trace/emqx_trace_api.erl | 40 ++++++++++--------- .../test/emqx_trace_SUITE.erl | 30 ++++++++++++-- 3 files changed, 55 insertions(+), 28 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index 252da8c0f..ad7112707 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -53,6 +53,7 @@ -ifdef(TEST). -export([ log_file/2 , create_table/0 + , find_closest_time/2 ]). -endif. @@ -288,15 +289,15 @@ get_enable_trace() -> find_closest_time(Traces, Now) -> Sec = lists:foldl( - fun(#?TRACE{start_at = Start, end_at = End}, Closest) - when Start >= Now andalso Now < End -> %% running - min(End - Now, Closest); - (#?TRACE{start_at = Start}, Closest) when Start < Now -> %% waiting - min(Now - Start, Closest); - (_, Closest) -> Closest %% finished + fun(#?TRACE{start_at = Start, end_at = End, enable = true}, Closest) -> + min(closest(End, Now, Closest), closest(Start, Now, Closest)); + (_, Closest) -> Closest end, 60 * 15, Traces), timer:seconds(Sec). +closest(Time, Now, Closest) when Now >= Time -> Closest; +closest(Time, Now, Closest) -> min(Time - Now, Closest). + disable_finished([]) -> ok; disable_finished(Traces) -> transaction(fun() -> diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index d2bca542b..0e298698e 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -135,7 +135,7 @@ stream_log_file(#{name := Name}, Params) -> {ok, Node} -> Position = binary_to_integer(Position0), Bytes = binary_to_integer(Bytes0), - case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of + case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of {ok, Bin} -> Meta = #{<<"position">> => Position + byte_size(Bin), <<"bytes">> => Bytes}, {ok, #{meta => Meta, items => Bin}}; @@ -143,7 +143,7 @@ stream_log_file(#{name := Name}, Params) -> Meta = #{<<"position">> => Size, <<"bytes">> => Bytes}, {ok, #{meta => Meta, items => <<"">>}}; {error, Reason} -> - logger:log(error, "read_file_failed by ~p", [{Node, Name, Reason, Position, Bytes}]), + logger:log(error, "read_file_failed ~p", [{Node, Name, Reason, Position, Bytes}]), {error, Reason}; {badrpc, nodedown} -> {error, "BadRpc node down"} @@ -174,21 +174,24 @@ read_trace_file(Name, Position, Limit) -> end. read_file(Path, Offset, Bytes) -> - {ok, IoDevice} = file:open(Path, [read, raw, binary]), - try - _ = case Offset of - 0 -> ok; - _ -> file:position(IoDevice, {bof, Offset}) - end, - case file:read(IoDevice, Bytes) of - {ok, Bin} -> {ok, Bin}; - {error, Reason} -> {error, Reason}; - eof -> - {ok, #file_info{size = Size}} = file:read_file_info(IoDevice), - {eof, Size} - end - after - file:close(IoDevice) + case file:open(Path, [read, raw, binary]) of + {ok, IoDevice} -> + try + _ = case Offset of + 0 -> ok; + _ -> file:position(IoDevice, {bof, Offset}) + end, + case file:read(IoDevice, Bytes) of + {ok, Bin} -> {ok, Bin}; + {error, Reason} -> {error, Reason}; + eof -> + {ok, #file_info{size = Size}} = file:read_file_info(IoDevice), + {eof, Size} + end + after + file:close(IoDevice) + end; + {error, Reason} -> {error, Reason} end. to_node(Node) -> @@ -204,7 +207,6 @@ collect_file_size(Nodes, FileName, AllFiles) -> end, #{}, Nodes). status(false, _Start, _End, _Now) -> <<"stopped">>; -%% asynchronously create trace, we should wait 1 seconds -status(true, Start, _End, Now) when Now < Start + 2 -> <<"waiting">>; +status(true, Start, _End, Now) when Now < Start -> <<"waiting">>; status(true, _Start, End, Now) when Now >= End -> <<"stopped">>; status(true, _Start, _End, _Now) -> <<"running">>. diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index 4f33e5b7f..e20fb6957 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -196,9 +196,9 @@ t_update_enable(_Config) -> t_load_state(_Config) -> Now = erlang:system_time(second), - Running = [{<<"name">>, <<"Running">>}, {<<"type">>, <<"topic">>}, - {<<"topic">>, <<"/x/y/1">>}, {<<"start_at">>, to_rfc3339(Now - 1)}, - {<<"end_at">>, to_rfc3339(Now + 2)}], + Running = #{name => <<"Running">>, type => <<"topic">>, + topic => <<"/x/y/1">>, start_at => to_rfc3339(Now - 1), + end_at => to_rfc3339(Now + 2)}, Waiting = [{<<"name">>, <<"Waiting">>}, {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/2">>}, {<<"start_at">>, to_rfc3339(Now + 3)}, {<<"end_at">>, to_rfc3339(Now + 8)}], @@ -300,6 +300,30 @@ t_download_log(_Config) -> ok = emqtt:disconnect(Client), ok. +t_find_closed_time(_Config) -> + DefaultMs = 60 * 15000, + Now = erlang:system_time(second), + Traces2 = [], + ?assertEqual(DefaultMs, emqx_trace:find_closest_time(Traces2, Now)), + Traces3 = [#emqx_trace{name = <<"disable">>, start_at = Now + 1, + end_at = Now + 2, enable = false}], + ?assertEqual(DefaultMs, emqx_trace:find_closest_time(Traces3, Now)), + Traces4 = [#emqx_trace{name = <<"running">>, start_at = Now, end_at = Now + 10, enable = true}], + ?assertEqual(10000, emqx_trace:find_closest_time(Traces4, Now)), + Traces5 = [#emqx_trace{name = <<"waiting">>, start_at = Now + 2, + end_at = Now + 10, enable = true}], + ?assertEqual(2000, emqx_trace:find_closest_time(Traces5, Now)), + Traces = [ + #emqx_trace{name = <<"waiting">>, start_at = Now + 1, end_at = Now + 2, enable = true}, + #emqx_trace{name = <<"running0">>, start_at = Now, end_at = Now + 5, enable = true}, + #emqx_trace{name = <<"running1">>, start_at = Now - 1, end_at = Now + 1, enable = true}, + #emqx_trace{name = <<"finished">>, start_at = Now - 2, end_at = Now - 1, enable = true}, + #emqx_trace{name = <<"waiting">>, start_at = Now + 1, end_at = Now + 1, enable = true}, + #emqx_trace{name = <<"stopped">>, start_at = Now, end_at = Now + 10, enable = false} + ], + ?assertEqual(1000, emqx_trace:find_closest_time(Traces, Now)), + ok. + to_rfc3339(Second) -> list_to_binary(calendar:system_time_to_rfc3339(Second)). From e651becd99fc008762a32734981c9ca2e6555f8e Mon Sep 17 00:00:00 2001 From: lafirest Date: Wed, 8 Dec 2021 15:05:39 +0800 Subject: [PATCH 103/104] fix(emqx_slow_subs): fix threshold related bugs 1. limit the interval between calling hooks 2. improve the code of update threshold --- .../src/emqx_slow_subs/emqx_slow_subs.erl | 16 +++++++++++----- lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl | 2 -- .../emqx_message_latency_stats.erl | 4 ++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl index e96c2a907..11d25380e 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -151,8 +151,8 @@ init_topk_tab() -> init([Conf]) -> notice_tick(Conf), expire_tick(Conf), - MaxSize = get_value(top_k_num, Conf), - load(MaxSize), + update_threshold(Conf), + load(Conf), {ok, #{config => Conf, last_tick_at => ?NOW, enable => true}}. @@ -163,8 +163,8 @@ handle_call({enable, Enable}, _From, IsEnable -> State; true -> - MaxSize = get_value(max_topk_num, Cfg), - load(MaxSize), + update_threshold(Cfg), + load(Cfg), State#{enable := true}; _ -> unload(), @@ -274,7 +274,8 @@ publish(TickTime, Cfg, Notices) -> _ = emqx_broker:safe_publish(Msg), ok. -load(MaxSize) -> +load(Cfg) -> + MaxSize = get_value(top_k_num, Cfg), _ = emqx:hook('message.slow_subs_stats', fun ?MODULE:on_stats_update/2, [#{max_size => MaxSize}]), @@ -319,3 +320,8 @@ try_insert_to_topk(MaxSize, Index, Latency, Type, Ts) -> ets:delete(?TOPK_TAB, First) end end. + +update_threshold(Conf) -> + Threshold = proplists:get_value(threshold, Conf), + _ = emqx_message_latency_stats:update_threshold(Threshold), + ok. diff --git a/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl b/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl index 1f5ebebf6..b9117fe8b 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl @@ -37,8 +37,6 @@ -spec(load(list()) -> ok). load(Env) -> - Threshold = proplists:get_value(threshold, Env), - _ = emqx_message_latency_stats:update_threshold(Threshold), emqx_mod_sup:start_child(?LIB, worker, [Env]), ok. diff --git a/src/emqx_slow_subs/emqx_message_latency_stats.erl b/src/emqx_slow_subs/emqx_message_latency_stats.erl index 49f0c8f1e..1d6158d59 100644 --- a/src/emqx_slow_subs/emqx_message_latency_stats.erl +++ b/src/emqx_slow_subs/emqx_message_latency_stats.erl @@ -98,6 +98,10 @@ call_hook(_, _, _, Latency, S) when Latency =< ?MINIMUM_THRESHOLD -> S; +call_hook(_, Now, _, _, #{last_access_time := LIT} = S) + when Now =< LIT + ?MINIMUM_INSERT_INTERVAL -> + S; + call_hook(ClientId, Now, Type, Latency, #{last_insert_value := LIV} = Stats) -> case Latency =< get_threshold() of true -> From d435f1211ee08e8b2083e7ab3699ff3384917e04 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 8 Dec 2021 10:33:57 -0300 Subject: [PATCH 104/104] fix(live_conn): fix live connection count on race condition When multiple clients try to connect concurrently using the same client ID, they all call `emqx_channel:ensure_connected`, increasing the live connection count, but only one will successfully acquire the lock for that client ID. This means that all other clients that increased the live connection count will not get to call neither `emqx_channel:ensure_disconnected` nor be monitored for `DOWN` messages, effectively causing a count leak. By moving the increment to `emqx_cm:register_channel`, which is only called inside the lock, we can remove this leakage. Also, during the handling of `DOWN` messages, we now iterate over all channel PIDs returned by `eqmx_misc:drain_down`, since it could be that one or more PIDs are not contained in the `pmon` state. --- src/emqx_channel.erl | 2 - src/emqx_cm.erl | 7 +--- test/emqx_broker_SUITE.erl | 75 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 7 deletions(-) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index bad053845..5524a391d 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -1548,8 +1548,6 @@ ensure_connected(Channel = #channel{conninfo = ConnInfo, clientinfo = ClientInfo}) -> NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, ok = run_hooks('client.connected', [ClientInfo, NConnInfo]), - ChanPid = self(), - emqx_cm:mark_channel_connected(ChanPid), Channel#channel{conninfo = NConnInfo, conn_state = connected }. diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index c54089e41..1c6d4080a 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -137,6 +137,7 @@ register_channel(ClientId, ChanPid, #{conn_mod := ConnMod}) when is_pid(ChanPid) true = ets:insert(?CHAN_TAB, Chan), true = ets:insert(?CHAN_CONN_TAB, {Chan, ConnMod}), ok = emqx_cm_registry:register_channel(Chan), + mark_channel_connected(ChanPid), cast({registered, Chan}). %% @doc Unregister a channel. @@ -465,11 +466,7 @@ handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon} ?tp(emqx_cm_process_down, #{pid => Pid, reason => _Reason}), ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), - lists:foreach( - fun({ChanPid, _ClientID}) -> - mark_channel_disconnected(ChanPid) - end, - Items), + lists:foreach(fun mark_channel_disconnected/1, ChanPids), ok = emqx_pool:async_submit(fun lists:foreach/2, [fun ?MODULE:clean_down/1, Items]), {noreply, State#{chan_pmon := PMon1}}; handle_info(Info, State) -> diff --git a/test/emqx_broker_SUITE.erl b/test/emqx_broker_SUITE.erl index 57f0d6acf..d44bb3412 100644 --- a/test/emqx_broker_SUITE.erl +++ b/test/emqx_broker_SUITE.erl @@ -37,6 +37,7 @@ groups() -> TCs = emqx_ct:all(?MODULE), ConnClientTCs = [ t_connected_client_count_persistent , t_connected_client_count_anonymous + , t_connected_client_count_transient_takeover , t_connected_client_stats ], OtherTCs = TCs -- ConnClientTCs, @@ -451,6 +452,80 @@ t_connected_client_count_anonymous({'end', _Config}) -> snabbkaffe:stop(), ok. +t_connected_client_count_transient_takeover({init, Config}) -> + ok = snabbkaffe:start_trace(), + process_flag(trap_exit, true), + Config; +t_connected_client_count_transient_takeover(Config) when is_list(Config) -> + ConnFun = ?config(conn_fun, Config), + ClientID = <<"clientid">>, + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% we spawn several clients simultaneously to cause the race + %% condition for the client id lock + NumClients = 20, + {ok, {ok, [_, _]}} = + wait_for_events( + fun() -> + lists:foreach( + fun(_) -> + spawn( + fun() -> + {ok, ConnPid} = + emqtt:start_link([ {clean_start, true} + , {clientid, ClientID} + | Config]), + %% don't assert the result: most of them fail + %% during the race + emqtt:ConnFun(ConnPid), + ok + end), + ok + end, + lists:seq(1, NumClients)) + end, + %% there can be only one channel that wins the race for the + %% lock for this client id. we also expect a decrement + %% event because the client dies along with the ephemeral + %% process. + [ emqx_cm_connected_client_count_inc + , emqx_cm_connected_client_count_dec + ], + 1000), + %% Since more than one pair of inc/dec may be emitted, we need to + %% wait for full stabilization + timer:sleep(100), + %% It must be 0 again because we spawn-linked the clients in + %% ephemeral processes above, and all should be dead now. + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% connecting again + {ok, ConnPid1} = emqtt:start_link([ {clean_start, true} + , {clientid, ClientID} + | Config + ]), + {{ok, _}, {ok, [_]}} = + wait_for_events( + fun() -> emqtt:ConnFun(ConnPid1) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% abnormal exit of channel process + [ChanPid] = emqx_cm:all_channels(), + {ok, {ok, [_, _]}} = + wait_for_events( + fun() -> + exit(ChanPid, kill), + ok + end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + ok; +t_connected_client_count_transient_takeover({'end', _Config}) -> + snabbkaffe:stop(), + ok. + t_connected_client_stats({init, Config}) -> ok = supervisor:terminate_child(emqx_kernel_sup, emqx_stats), {ok, _} = supervisor:restart_child(emqx_kernel_sup, emqx_stats),